The package rpms/pacemaker.git has added or updated architecture specific content in its
spec file (ExclusiveArch/ExcludeArch or %ifarch/%ifnarch) in commit(s):
https://src.fedoraproject.org/cgit/rpms/pacemaker.git/commit/?id=5603f192....
Change:
+ExclusiveArch: aarch64 i686 ppc64le s390x x86_64
Thanks.
Full change:
============
commit 5603f1922d76984e428988af1575ffd06e0c322f
Author: Klaus Wenninger <klaus.wenninger(a)aon.at>
Date: Mon Jul 5 18:38:43 2021 +0200
* Mon Jul 5 2021 Klaus Wenninger <kwenning(a)redhat.com> - 2.1.0-6
- synced/merged with CS9 spec-file for current 2.1.0-release build
* Tue Jun 8 2021 Klaus Wenninger <kwenning(a)redhat.com> - 2.1.0-0.5.rc3
- silence f33 s390x build complaining about possible format-trucation
diff --git a/001-ping-agent.patch b/001-ping-agent.patch
new file mode 100644
index 0000000..89fe41a
--- /dev/null
+++ b/001-ping-agent.patch
@@ -0,0 +1,225 @@
+From c6ee0973522268ed7b3241cf0ec2e06398444114 Mon Sep 17 00:00:00 2001
+From: Grace Chin <gchin(a)redhat.com>
+Date: Tue, 4 May 2021 12:02:17 -0400
+Subject: [PATCH 1/4] Remove deprecated attrd_options
+
+---
+ extra/resources/ping | 11 +++--------
+ 1 file changed, 3 insertions(+), 8 deletions(-)
+
+diff --git a/extra/resources/ping b/extra/resources/ping
+index 3cf8dfe..2e93f22 100755
+--- a/extra/resources/ping
++++ b/extra/resources/ping
+@@ -178,7 +178,7 @@ ping_stop() {
+
+ rm -f "${OCF_RESKEY_pidfile}"
+
+- attrd_updater -D -n "$OCF_RESKEY_name" -d "$OCF_RESKEY_dampen"
$attrd_options
++ attrd_updater -D -n "$OCF_RESKEY_name" -d "$OCF_RESKEY_dampen"
+
+ return $OCF_SUCCESS
+ }
+@@ -302,9 +302,9 @@ ping_update() {
+
+ score=$(expr $active \* $OCF_RESKEY_multiplier)
+ if [ "$__OCF_ACTION" = "start" ] ; then
+- attrd_updater -n "$OCF_RESKEY_name" -B "$score" -d
"$OCF_RESKEY_dampen" $attrd_options
++ attrd_updater -n "$OCF_RESKEY_name" -B "$score" -d
"$OCF_RESKEY_dampen"
+ else
+- attrd_updater -n "$OCF_RESKEY_name" -v "$score" -d
"$OCF_RESKEY_dampen" $attrd_options
++ attrd_updater -n "$OCF_RESKEY_name" -v "$score" -d
"$OCF_RESKEY_dampen"
+ fi
+ rc=$?
+ case $rc in
+@@ -396,11 +396,6 @@ case "${OCF_RESKEY_debug}" in
+ ;;
+ esac
+
+-attrd_options='-q'
+-if [ "${OCF_RESKEY_debug}" = "true" ]; then
+- attrd_options=''
+-fi
+-
+ case "$__OCF_ACTION" in
+ meta-data) meta_data
+ exit $OCF_SUCCESS
+--
+1.8.3.1
+
+
+From 6d6c4691cf0970059689856c354daf9e098b4451 Mon Sep 17 00:00:00 2001
+From: Grace Chin <gchin(a)redhat.com>
+Date: Tue, 4 May 2021 14:50:37 -0400
+Subject: [PATCH 2/4] Replace debug values, true and false, with 0 and 1
+
+---
+ extra/resources/ping | 8 ++++----
+ 1 file changed, 4 insertions(+), 4 deletions(-)
+
+diff --git a/extra/resources/ping b/extra/resources/ping
+index 2e93f22..fee019b 100755
+--- a/extra/resources/ping
++++ b/extra/resources/ping
+@@ -24,7 +24,7 @@
+ : ${OCF_RESKEY_dampen:="5s"}
+ : ${OCF_RESKEY_attempts:="3"}
+ : ${OCF_RESKEY_multiplier:="1"}
+-: ${OCF_RESKEY_debug:="false"}
++: ${OCF_RESKEY_debug:="0"}
+ : ${OCF_RESKEY_failure_score:="0"}
+ : ${OCF_RESKEY_use_fping:="1"}
+ : ${OCF_RESKEY_host_list:=""}
+@@ -152,7 +152,7 @@ END
+
+ ping_conditional_log() {
+ level="$1"; shift
+- if [ "${OCF_RESKEY_debug}" = "true" ]; then
++ if [ $OCF_RESKEY_debug -gt 0 ]; then
+ ocf_log "$level" "$*"
+ fi
+ }
+@@ -388,8 +388,8 @@ fi
+
+ # Check the debug option
+ case "${OCF_RESKEY_debug}" in
+- true|True|TRUE|1) OCF_RESKEY_debug=true;;
+- false|False|FALSE|0) OCF_RESKEY_debug=false;;
++ true|True|TRUE|1) OCF_RESKEY_debug=0;;
++ false|False|FALSE|0) OCF_RESKEY_debug=1;;
+ *)
+ ocf_log warn "Value for 'debug' is incorrect. Please specify
'true' or 'false' not: ${OCF_RESKEY_debug}"
+ OCF_RESKEY_debug=false
+--
+1.8.3.1
+
+
+From a886a31056b6aca764c6911f5432af2c5ebf51df Mon Sep 17 00:00:00 2001
+From: Grace Chin <gchin(a)redhat.com>
+Date: Tue, 11 May 2021 11:04:50 -0400
+Subject: [PATCH 3/4] Add verbose debug mode which logs ping and fping output
+ when set
+
+---
+ extra/resources/ping | 19 ++++++++++++++-----
+ 1 file changed, 14 insertions(+), 5 deletions(-)
+
+diff --git a/extra/resources/ping b/extra/resources/ping
+index fee019b..cc796af 100755
+--- a/extra/resources/ping
++++ b/extra/resources/ping
+@@ -249,10 +249,13 @@ fping_check() {
+
+ case $rc in
+ 0)
++ if [ $OCF_RESKEY_debug -gt 1 ]; then
++ ping_conditional_log info "$output"
++ fi
+ ;;
+ 1)
+ for h in $(echo "$output" | grep "is unreachable" | awk
'{print $1}'); do
+- ping_conditional_log warn "$h is inactive"
++ ping_conditional_log warn "$h is inactive: $output"
+ done
+ ;;
+ *)
+@@ -282,7 +285,12 @@ ping_check() {
+ p_out=$($p_exe $p_args $OCF_RESKEY_options $host 2>&1); rc=$?
+
+ case $rc in
+- 0) active=$(expr $active + 1);;
++ 0)
++ active=$(expr $active + 1)
++ if [ $OCF_RESKEY_debug -gt 1 ]; then
++ ping_conditional_log info "$p_out"
++ fi
++ ;;
+ 1) ping_conditional_log warn "$host is inactive: $p_out";;
+ *) ocf_log err "Unexpected result for '$p_exe $p_args
$OCF_RESKEY_options $host' $rc: $p_out";;
+ esac
+@@ -388,10 +396,11 @@ fi
+
+ # Check the debug option
+ case "${OCF_RESKEY_debug}" in
+- true|True|TRUE|1) OCF_RESKEY_debug=0;;
+- false|False|FALSE|0) OCF_RESKEY_debug=1;;
++ true|True|TRUE|1) OCF_RESKEY_debug=1;;
++ false|False|FALSE|0) OCF_RESKEY_debug=0;;
++ verbose|Verbose|VERBOSE|2) OCF_RESKEY_debug=2;;
+ *)
+- ocf_log warn "Value for 'debug' is incorrect. Please specify
'true' or 'false' not: ${OCF_RESKEY_debug}"
++ ocf_log warn "Value for 'debug' is incorrect. Please specify
'true', 'false', or 'verbose', not: ${OCF_RESKEY_debug}"
+ OCF_RESKEY_debug=false
+ ;;
+ esac
+--
+1.8.3.1
+
+
+From 460043f133ced80e923b1290af70502a72deb7f8 Mon Sep 17 00:00:00 2001
+From: Grace Chin <gchin(a)redhat.com>
+Date: Tue, 11 May 2021 11:07:05 -0400
+Subject: [PATCH 4/4] Improve variable names
+
+---
+ extra/resources/ping | 20 ++++++++++----------
+ 1 file changed, 10 insertions(+), 10 deletions(-)
+
+diff --git a/extra/resources/ping b/extra/resources/ping
+index cc796af..9763b60 100755
+--- a/extra/resources/ping
++++ b/extra/resources/ping
+@@ -244,22 +244,22 @@ fping_check() {
+ timeout=$(expr $OCF_RESKEY_timeout \* 1000 / $OCF_RESKEY_attempts)
+
+ cmd="$p_exe -r $OCF_RESKEY_attempts -t $timeout -B 1.0 $OCF_RESKEY_options
$OCF_RESKEY_host_list"
+- output=$($cmd 2>&1); rc=$?
+- active=$(echo "$output" | grep "is alive" | wc -l)
++ fping_output=$($cmd 2>&1); rc=$?
++ active=$(echo "$fping_output" | grep "is alive" | wc -l)
+
+ case $rc in
+ 0)
+ if [ $OCF_RESKEY_debug -gt 1 ]; then
+- ping_conditional_log info "$output"
++ ping_conditional_log info "$fping_output"
+ fi
+ ;;
+ 1)
+- for h in $(echo "$output" | grep "is unreachable" | awk
'{print $1}'); do
+- ping_conditional_log warn "$h is inactive: $output"
++ for h in $(echo "$fping_output" | grep "is unreachable"
| awk '{print $1}'); do
++ ping_conditional_log warn "$h is inactive: $fping_output"
+ done
+ ;;
+ *)
+- ocf_log err "Unexpected result for '$cmd' $rc: $(echo
"$output" | tr '\n' ';')"
++ ocf_log err "Unexpected result for '$cmd' $rc: $(echo
"$fping_output" | tr '\n' ';')"
+ ;;
+ esac
+
+@@ -282,17 +282,17 @@ ping_check() {
+ *:*) p_exe=ping6
+ esac
+
+- p_out=$($p_exe $p_args $OCF_RESKEY_options $host 2>&1); rc=$?
++ ping_output=$($p_exe $p_args $OCF_RESKEY_options $host 2>&1); rc=$?
+
+ case $rc in
+ 0)
+ active=$(expr $active + 1)
+ if [ $OCF_RESKEY_debug -gt 1 ]; then
+- ping_conditional_log info "$p_out"
++ ping_conditional_log info "$ping_output"
+ fi
+ ;;
+- 1) ping_conditional_log warn "$host is inactive: $p_out";;
+- *) ocf_log err "Unexpected result for '$p_exe $p_args
$OCF_RESKEY_options $host' $rc: $p_out";;
++ 1) ping_conditional_log warn "$host is inactive: $ping_output";;
++ *) ocf_log err "Unexpected result for '$p_exe $p_args
$OCF_RESKEY_options $host' $rc: $ping_output";;
+ esac
+ done
+ return $active
+--
+1.8.3.1
+
diff --git a/002-pacemakerd-options.patch b/002-pacemakerd-options.patch
new file mode 100644
index 0000000..56941ec
--- /dev/null
+++ b/002-pacemakerd-options.patch
@@ -0,0 +1,451 @@
+From 0d40ebf10b1794ece2c5c9768ea7222d3834d3b3 Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Thu, 13 May 2021 11:42:18 -0400
+Subject: [PATCH 1/4] Build: Use a different variable to find man page
+ includes.
+
+With other programs outside of the tools directory being converted to
+use glib for command line handling, their includes are not going to be
+in tools/. So we need to use a different autoconf variable to find
+them.
+---
+ mk/common.mk | 6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/mk/common.mk b/mk/common.mk
+index b247670..aa59feb 100644
+--- a/mk/common.mk
++++ b/mk/common.mk
+@@ -1,5 +1,5 @@
+ #
+-# Copyright 2014-2020 the Pacemaker project contributors
++# Copyright 2014-2021 the Pacemaker project contributors
+ #
+ # The version control history for this file may have further details.
+ #
+@@ -68,11 +68,11 @@ HELP2MAN_ARGS = -N --section 8 --name "Part of the Pacemaker
cluster resource ma
+ # and all wrappers to C code.
+ %.8: % $(MAN8DEPS)
+ $(AM_V_at)chmod a+x $(abs_builddir)/$<
+- $(AM_V_MAN)if [ -f $(top_srcdir)/tools/$@.inc ]; then \
++ $(AM_V_MAN)if [ -f $(abs_srcdir)/$@.inc ]; then \
+ PATH=$(abs_builddir):$$PATH $(HELP2MAN) $(HELP2MAN_ARGS) \
+ -h --help-all \
+ --no-discard-stderr \
+- -i $(top_srcdir)/tools/$@.inc $(abs_builddir)/$< \
++ -i $(abs_srcdir)/$@.inc $(abs_builddir)/$< \
+ | sed -f $(top_srcdir)/tools/fix-manpages > $@ ; \
+ else \
+ PATH=$(abs_builddir):$$PATH $(HELP2MAN) $(HELP2MAN_ARGS) \
+--
+1.8.3.1
+
+
+From c7ab1d901bcbbf0137277e783e072777ca2f82d9 Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Thu, 13 May 2021 11:44:16 -0400
+Subject: [PATCH 2/4] Refactor: daemons: Remove the pid_file variable from
+ pacemakerd.
+
+It's never used anywhere.
+---
+ daemons/pacemakerd/pacemakerd.c | 3 ---
+ 1 file changed, 3 deletions(-)
+
+diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c
+index 8ec9708..03d688e 100644
+--- a/daemons/pacemakerd/pacemakerd.c
++++ b/daemons/pacemakerd/pacemakerd.c
+@@ -27,8 +27,7 @@
+
+ static crm_trigger_t *shutdown_trigger = NULL;
+ static crm_trigger_t *startup_trigger = NULL;
+-static const char *pid_file = PCMK_RUN_DIR "/pacemaker.pid";
+
+ /* state we report when asked via pacemakerd-api status-ping */
+ static const char *pacemakerd_state = XML_PING_ATTR_PACEMAKERDSTATE_INIT;
+ static gboolean running_with_sbd = FALSE; /* local copy */
+@@ -224,7 +222,6 @@ main(int argc, char **argv)
+ /* Legacy */
+ break;
+ case 'p':
+- pid_file = optarg;
+ break;
+ case 's':
+ pcmk__set_env_option("node_start_state",
"standby");
+--
+1.8.3.1
+
+
+From 98990eed9f6a5dbde7c8a5aa0783e93d5479295b Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Thu, 13 May 2021 13:14:38 -0400
+Subject: [PATCH 3/4] Refactor: daemons: Use glib for command line handling in
+ pacemakerd.
+
+---
+ daemons/pacemakerd/Makefile.am | 2 +
+ daemons/pacemakerd/pacemakerd.8.inc | 5 +
+ daemons/pacemakerd/pacemakerd.c | 195 ++++++++++++++++++------------------
+ 3 files changed, 102 insertions(+), 100 deletions(-)
+ create mode 100644 daemons/pacemakerd/pacemakerd.8.inc
+
+diff --git a/daemons/pacemakerd/Makefile.am b/daemons/pacemakerd/Makefile.am
+index cc657f5..84517a3 100644
+--- a/daemons/pacemakerd/Makefile.am
++++ b/daemons/pacemakerd/Makefile.am
+@@ -15,6 +15,8 @@ if BUILD_SYSTEMD
+ systemdsystemunit_DATA = pacemaker.service
+ endif
+
++EXTRA_DIST = pacemakerd.8.inc
++
+ ## SOURCES
+
+ noinst_HEADERS = pacemakerd.h
+diff --git a/daemons/pacemakerd/pacemakerd.8.inc b/daemons/pacemakerd/pacemakerd.8.inc
+new file mode 100644
+index 0000000..902af4e
+--- /dev/null
++++ b/daemons/pacemakerd/pacemakerd.8.inc
+@@ -0,0 +1,5 @@
++[synopsis]
++pacemakerd [options]
++
++/subsidiary Pacemaker daemons/
++.SH OPTIONS
+diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c
+index 03d688e..ce194bf 100644
+--- a/daemons/pacemakerd/pacemakerd.c
++++ b/daemons/pacemakerd/pacemakerd.c
+@@ -23,12 +23,54 @@
+ #include <crm/msg_xml.h>
+ #include <crm/common/ipc_internal.h>
+ #include <crm/common/mainloop.h>
++#include <crm/common/cmdline_internal.h>
+ #include <crm/cluster/internal.h>
+ #include <crm/cluster.h>
+
+ #include <dirent.h>
+ #include <ctype.h>
+
++#define SUMMARY "pacemakerd - primary Pacemaker daemon that launches and monitors
all subsidiary Pacemaker daemons"
++
++struct {
++ gboolean features;
++ gboolean foreground;
++ gboolean shutdown;
++ gboolean standby;
++} options;
++
++static gboolean
++pid_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
++ return TRUE;
++}
++
++static gboolean
++standby_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err)
{
++ options.standby = TRUE;
++ pcmk__set_env_option("node_start_state", "standby");
++ return TRUE;
++}
++
++static GOptionEntry entries[] = {
++ { "features", 'F', 0, G_OPTION_ARG_NONE, &options.features,
++ "Display full version and list of features Pacemaker was built with",
++ NULL },
++ { "foreground", 'f', 0, G_OPTION_ARG_NONE,
&options.foreground,
++ "(Ignored) Pacemaker always runs in the foreground",
++ NULL },
++ { "pid-file", 'p', 0, G_OPTION_ARG_CALLBACK, pid_cb,
++ "(Ignored) Daemon pid file location",
++ "FILE" },
++ { "shutdown", 'S', 0, G_OPTION_ARG_NONE, &options.shutdown,
++ "Instruct Pacemaker to shutdown on this machine",
++ NULL },
++ { "standby", 's', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
standby_cb,
++ "Start node in standby state",
++ NULL },
++
++ { NULL }
++};
++
+ static gboolean fatal_error = FALSE;
+ static GMainLoop *mainloop = NULL;
+ static bool global_keep_tracking = false;
+@@ -642,49 +685,6 @@ pcmk_sigquit(int nsig)
+ .connection_destroyed = pcmk_ipc_destroy
+ };
+
+-static pcmk__cli_option_t long_options[] = {
+- // long option, argument type, storage, short option, description, flags
+- {
+- "help", no_argument, NULL, '?',
+- "\tThis text", pcmk__option_default
+- },
+- {
+- "version", no_argument, NULL, '$',
+- "\tVersion information", pcmk__option_default
+- },
+- {
+- "verbose", no_argument, NULL, 'V',
+- "\tIncrease debug output", pcmk__option_default
+- },
+- {
+- "shutdown", no_argument, NULL, 'S',
+- "\tInstruct Pacemaker to shutdown on this machine",
pcmk__option_default
+- },
+- {
+- "features", no_argument, NULL, 'F',
+- "\tDisplay full version and list of features Pacemaker was built
with",
+- pcmk__option_default
+- },
+- {
+- "-spacer-", no_argument, NULL, '-',
+- "\nAdditional Options:", pcmk__option_default
+- },
+- {
+- "foreground", no_argument, NULL, 'f',
+- "\t(Ignored) Pacemaker always runs in the foreground",
+- pcmk__option_default
+- },
+- {
+- "pid-file", required_argument, NULL, 'p',
+- "\t(Ignored) Daemon pid file location", pcmk__option_default
+- },
+- {
+- "standby", no_argument, NULL, 's',
+- "\tStart node in standby state", pcmk__option_default
+- },
+- { 0, 0, 0, 0 }
+-};
+-
+ static void
+ mcp_chown(const char *path, uid_t uid, gid_t gid)
+ {
+@@ -1168,83 +1211,66 @@ request_shutdown(crm_ipc_t *ipc)
+ return status;
+ }
+
++static GOptionContext *
++build_arg_context(pcmk__common_args_t *args) {
++ GOptionContext *context = NULL;
++
++ context = pcmk__build_arg_context(args, NULL, NULL, NULL);
++ pcmk__add_main_args(context, entries);
++ return context;
++}
++
+ int
+ main(int argc, char **argv)
+ {
+- int flag;
+- int argerr = 0;
++ crm_exit_t exit_code = CRM_EX_OK;
++
++ GError *error = NULL;
++
++ pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
++ gchar **processed_args = pcmk__cmdline_preproc(argv, "p");
++ GOptionContext *context = build_arg_context(args);
+
+- int option_index = 0;
+ bool old_instance_connected = false;
+- gboolean shutdown = FALSE;
+
+ crm_ipc_t *old_instance = NULL;
+ qb_ipcs_service_t *ipcs = NULL;
+
+ crm_log_preinit(NULL, argc, argv);
+- pcmk__set_cli_options(NULL, "[options]", long_options,
+- "primary Pacemaker daemon that launches and "
+- "monitors all subsidiary Pacemaker daemons");
+ mainloop_add_signal(SIGHUP, pcmk_ignore);
+ mainloop_add_signal(SIGQUIT, pcmk_sigquit);
+
+- while (1) {
+- flag = pcmk__next_cli_option(argc, argv, &option_index, NULL);
+- if (flag == -1)
+- break;
+-
+- switch (flag) {
+- case 'V':
+- crm_bump_log_level(argc, argv);
+- break;
+- case 'f':
+- /* Legacy */
+- break;
+- case 'p':
+- break;
+- case 's':
+- pcmk__set_env_option("node_start_state",
"standby");
+- break;
+- case '$':
+- case '?':
+- pcmk__cli_help(flag, CRM_EX_OK);
+- break;
+- case 'S':
+- shutdown = TRUE;
+- break;
+- case 'F':
+- printf("Pacemaker %s (Build: %s)\n Supporting v%s: %s\n",
PACEMAKER_VERSION, BUILD_VERSION,
+- CRM_FEATURE_SET, CRM_FEATURES);
+- crm_exit(CRM_EX_OK);
+- default:
+- printf("Argument code 0%o (%c) is not (?yet?) supported\n",
flag, flag);
+- ++argerr;
+- break;
+- }
++ if (!g_option_context_parse_strv(context, &processed_args, &error)) {
++ exit_code = CRM_EX_USAGE;
++ goto done;
+ }
+
+- if (optind < argc) {
+- printf("non-option ARGV-elements: ");
+- while (optind < argc)
+- printf("%s ", argv[optind++]);
+- printf("\n");
+- }
+- if (argerr) {
+- pcmk__cli_help('?', CRM_EX_USAGE);
++ if (options.features) {
++ printf("Pacemaker %s (Build: %s)\n Supporting v%s: %s\n",
PACEMAKER_VERSION, BUILD_VERSION,
++ CRM_FEATURE_SET, CRM_FEATURES);
++ exit_code = CRM_EX_OK;
++ goto done;
+ }
+
++ if (args->version) {
++ g_strfreev(processed_args);
++ pcmk__free_arg_context(context);
++ /* FIXME: When pacemakerd is converted to use formatted output, this can go.
*/
++ pcmk__cli_help('v', CRM_EX_USAGE);
++ }
+
+ setenv("LC_ALL", "C", 1);
+
+ pcmk__set_env_option("mcp", "true");
+
++ pcmk__cli_init_logging("pacemakerd", args->verbosity);
+ crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
+
+ crm_debug("Checking for existing Pacemaker instance");
+ old_instance = crm_ipc_new(CRM_SYSTEM_MCP, 0);
+ old_instance_connected = crm_ipc_connect(old_instance);
+
+- if (shutdown) {
++ if (options.shutdown) {
+ if (old_instance_connected) {
+ crm_exit(request_shutdown(old_instance));
+ } else {
+@@ -1253,22 +1279,25 @@ main(int argc, char **argv)
+ "Pacemaker instance: %s", strerror(errno));
+ crm_ipc_close(old_instance);
+ crm_ipc_destroy(old_instance);
+- crm_exit(CRM_EX_DISCONNECT);
++ exit_code = CRM_EX_DISCONNECT;
++ goto done;
+ }
+
+ } else if (old_instance_connected) {
+ crm_ipc_close(old_instance);
+ crm_ipc_destroy(old_instance);
+ crm_err("Aborting start-up because active Pacemaker instance found");
+- crm_exit(CRM_EX_FATAL);
++ exit_code = CRM_EX_FATAL;
++ goto done;
+ }
+
+ crm_ipc_close(old_instance);
+ crm_ipc_destroy(old_instance);
+
+ #ifdef SUPPORT_COROSYNC
+ if (mcp_read_config() == FALSE) {
+- crm_exit(CRM_EX_UNAVAILABLE);
++ exit_code = CRM_EX_UNAVAILABLE;
++ goto done;
+ }
+ #endif
+
+@@ -1292,7 +1321,8 @@ main(int argc, char **argv)
+ #ifdef SUPPORT_COROSYNC
+ /* Allows us to block shutdown */
+ if (!cluster_connect_cfg()) {
+- crm_exit(CRM_EX_PROTOCOL);
++ exit_code = CRM_EX_PROTOCOL;
++ goto done;
+ }
+ #endif
+
+@@ -1307,9 +1337,11 @@ main(int argc, char **argv)
+ case pcmk_rc_ok:
+ break;
+ case pcmk_rc_ipc_unauthorized:
+- crm_exit(CRM_EX_CANTCREAT);
++ exit_code = CRM_EX_CANTCREAT;
++ goto done;
+ default:
+- crm_exit(CRM_EX_FATAL);
++ exit_code = CRM_EX_FATAL;
++ goto done;
+ };
+
+ mainloop_add_signal(SIGTERM, pcmk_shutdown);
+@@ -1342,5 +1374,11 @@ main(int argc, char **argv)
+ #ifdef SUPPORT_COROSYNC
+ cluster_disconnect_cfg();
+ #endif
+- crm_exit(CRM_EX_OK);
++
++done:
++ g_strfreev(processed_args);
++ pcmk__free_arg_context(context);
++
++ pcmk__output_and_clear_error(error, NULL);
++ crm_exit(exit_code);
+ }
+--
+1.8.3.1
+
+
+From 8f7924fbb2a012bedcad59335b7bebc5020b26e3 Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Thu, 13 May 2021 13:27:13 -0400
+Subject: [PATCH 4/4] Low: pacemaker.service: Don't start pacemakerd with -f.
+
+This option is completely ignored by pacemakerd.
+---
+ daemons/pacemakerd/pacemaker.service.in | 2 +-
+ doc/sphinx/Clusters_from_Scratch/verification.rst | 2 +-
+ 2 files changed, 2 insertions(+), 2 deletions(-)
+
+diff --git a/daemons/pacemakerd/pacemaker.service.in
b/daemons/pacemakerd/pacemaker.service.in
+index b128ddc..0363a22 100644
+--- a/daemons/pacemakerd/pacemaker.service.in
++++ b/daemons/pacemakerd/pacemaker.service.in
+@@ -44,7 +44,7 @@ EnvironmentFile=-@CONFIGDIR@/pacemaker
+ EnvironmentFile=-@CONFIGDIR@/sbd
+ SuccessExitStatus=100
+
+-ExecStart=@sbindir@/pacemakerd -f
++ExecStart=@sbindir@/pacemakerd
+
+ # Systemd v227 and above can limit the number of processes spawned by a
+ # service. That is a bad idea for an HA cluster resource manager, so disable it
+diff --git a/doc/sphinx/Clusters_from_Scratch/verification.rst
b/doc/sphinx/Clusters_from_Scratch/verification.rst
+index 9d647f8..b7fa20e 100644
+--- a/doc/sphinx/Clusters_from_Scratch/verification.rst
++++ b/doc/sphinx/Clusters_from_Scratch/verification.rst
+@@ -103,7 +103,7 @@ the necessary processes are running:
+ 2 ? S 0:00 [kthreadd]
+ ...lots of processes...
+ 17121 ? SLsl 0:01 /usr/sbin/corosync -f
+- 17133 ? Ss 0:00 /usr/sbin/pacemakerd -f
++ 17133 ? Ss 0:00 /usr/sbin/pacemakerd
+ 17134 ? Ss 0:00 \_ /usr/libexec/pacemaker/pacemaker-based
+ 17135 ? Ss 0:00 \_ /usr/libexec/pacemaker/pacemaker-fenced
+ 17136 ? Ss 0:00 \_ /usr/libexec/pacemaker/pacemaker-execd
+--
+1.8.3.1
+
diff --git a/003-pacemakerd-output.patch b/003-pacemakerd-output.patch
new file mode 100644
index 0000000..167e22b
--- /dev/null
+++ b/003-pacemakerd-output.patch
@@ -0,0 +1,343 @@
+From 7c35387a9896cb968cf4087b5cbed94af44e1ea5 Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Fri, 14 May 2021 12:03:46 -0400
+Subject: [PATCH 1/5] Feature: daemons: Convert pacemakerd to formatted output.
+
+The main purpose of this is to finish getting pacemakerd moved off the
+existing command line handling code (pcmk__cli_help in particular) so
+that code can eventually be deprecated or removed. pacemakerd itself
+does fairly little printing.
+---
+ daemons/pacemakerd/pacemakerd.c | 58 ++++++++++++++++++++++++++++++-----------
+ 1 file changed, 43 insertions(+), 15 deletions(-)
+
+diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c
+index ce194bf..bd59729 100644
+--- a/daemons/pacemakerd/pacemakerd.c
++++ b/daemons/pacemakerd/pacemakerd.c
+@@ -25,6 +25,7 @@
+ #include <crm/common/ipc_internal.h>
+ #include <crm/common/mainloop.h>
+ #include <crm/common/cmdline_internal.h>
++#include <crm/common/output_internal.h>
+ #include <crm/cluster/internal.h>
+ #include <crm/cluster.h>
+
+@@ -37,6 +38,14 @@ struct {
+ gboolean standby;
+ } options;
+
++static pcmk__output_t *out = NULL;
++
++static pcmk__supported_format_t formats[] = {
++ PCMK__SUPPORTED_FORMAT_NONE,
++ PCMK__SUPPORTED_FORMAT_TEXT,
++ { NULL, NULL, NULL }
++};
++
+ static gboolean
+ pid_cb(const gchar *option_name, const gchar *optarg, gpointer data, GError **err) {
+ return TRUE;
+@@ -1167,10 +1176,10 @@ pacemakerd_event_cb(pcmk_ipc_api_t *pacemakerd_api,
+ }
+
+ static GOptionContext *
+-build_arg_context(pcmk__common_args_t *args) {
++build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
+ GOptionContext *context = NULL;
+
+- context = pcmk__build_arg_context(args, NULL, NULL, NULL);
++ context = pcmk__build_arg_context(args, "text", group, NULL);
+ pcmk__add_main_args(context, entries);
+ return context;
+ }
+@@ -1182,9 +1191,11 @@ main(int argc, char **argv)
+
+ GError *error = NULL;
+
++ int rc = pcmk_rc_ok;
++ GOptionGroup *output_group = NULL;
+ pcmk__common_args_t *args = pcmk__new_common_args(SUMMARY);
+ gchar **processed_args = pcmk__cmdline_preproc(argv, "p");
+- GOptionContext *context = build_arg_context(args);
++ GOptionContext *context = build_arg_context(args, &output_group);
+
+ bool old_instance_connected = false;
+
+@@ -1195,23 +1205,30 @@ main(int argc, char **argv)
+ mainloop_add_signal(SIGHUP, pcmk_ignore);
+ mainloop_add_signal(SIGQUIT, pcmk_sigquit);
+
++ pcmk__register_formats(output_group, formats);
+ if (!g_option_context_parse_strv(context, &processed_args, &error)) {
+ exit_code = CRM_EX_USAGE;
+ goto done;
+ }
+
++ rc = pcmk__output_new(&out, args->output_ty, args->output_dest, argv);
++ if (rc != pcmk_rc_ok) {
++ exit_code = CRM_EX_ERROR;
++ g_set_error(&error, PCMK__EXITC_ERROR, exit_code, "Error creating
output format %s: %s",
++ args->output_ty, pcmk_rc_str(rc));
++ goto done;
++ }
++
+ if (options.features) {
+- printf("Pacemaker %s (Build: %s)\n Supporting v%s: %s\n",
PACEMAKER_VERSION, BUILD_VERSION,
+- CRM_FEATURE_SET, CRM_FEATURES);
++ out->info(out, "Pacemaker %s (Build: %s)\n Supporting v%s: %s",
PACEMAKER_VERSION,
++ BUILD_VERSION, CRM_FEATURE_SET, CRM_FEATURES);
+ exit_code = CRM_EX_OK;
+ goto done;
+ }
+
+ if (args->version) {
+- g_strfreev(processed_args);
+- pcmk__free_arg_context(context);
+- /* FIXME: When pacemakerd is converted to use formatted output, this can go.
*/
+- pcmk__cli_help('v', CRM_EX_USAGE);
++ out->version(out, false);
++ goto done;
+ }
+
+ setenv("LC_ALL", "C", 1);
+@@ -1248,6 +1265,13 @@ main(int argc, char **argv)
+ crm_ipc_close(old_instance);
+ crm_ipc_destroy(old_instance);
+
++ /* Don't allow any accidental output after this point. */
++ if (out != NULL) {
++ out->finish(out, exit_code, true, NULL);
++ pcmk__output_free(out);
++ out = NULL;
++ }
++
+ #ifdef SUPPORT_COROSYNC
+ if (mcp_read_config() == FALSE) {
+ exit_code = CRM_EX_UNAVAILABLE;
+@@ -1333,6 +1357,11 @@ done:
+ g_strfreev(processed_args);
+ pcmk__free_arg_context(context);
+
+- pcmk__output_and_clear_error(error, NULL);
++ pcmk__output_and_clear_error(error, out);
++
++ if (out != NULL) {
++ out->finish(out, exit_code, true, NULL);
++ pcmk__output_free(out);
++ }
+ crm_exit(exit_code);
+ }
+--
+1.8.3.1
+
+
+From 35e6da64381fcb092d81ce16835cc28670b077cb Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Mon, 17 May 2021 10:04:04 -0400
+Subject: [PATCH 2/5] Features: daemons: Output the pacemakerd feature list in
+ XML.
+
+---
+ daemons/pacemakerd/pacemakerd.c | 45 ++++++++++++++++++++++++++++++++++++++---
+ 1 file changed, 42 insertions(+), 3 deletions(-)
+
+diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c
+index bd59729..93cf743 100644
+--- a/daemons/pacemakerd/pacemakerd.c
++++ b/daemons/pacemakerd/pacemakerd.c
+@@ -43,6 +43,42 @@ static pcmk__output_t *out = NULL;
+ static pcmk__supported_format_t formats[] = {
+ PCMK__SUPPORTED_FORMAT_NONE,
+ PCMK__SUPPORTED_FORMAT_TEXT,
++ PCMK__SUPPORTED_FORMAT_XML,
++ { NULL, NULL, NULL }
++};
++
++static int
++pacemakerd_features(pcmk__output_t *out, va_list args) {
++ out->info(out, "Pacemaker %s (Build: %s)\n Supporting v%s: %s",
PACEMAKER_VERSION,
++ BUILD_VERSION, CRM_FEATURE_SET, CRM_FEATURES);
++ return pcmk_rc_ok;
++}
++
++static int
++pacemakerd_features_xml(pcmk__output_t *out, va_list args) {
++ gchar **feature_list = g_strsplit(CRM_FEATURES, " ", 0);
++
++ pcmk__output_xml_create_parent(out, "pacemakerd",
++ "version", PACEMAKER_VERSION,
++ "build", BUILD_VERSION,
++ "feature_set", CRM_FEATURE_SET,
++ NULL);
++ out->begin_list(out, NULL, NULL, "features");
++
++ for (char **s = feature_list; *s != NULL; s++) {
++ pcmk__output_create_xml_text_node(out, "feature", *s);
++ }
++
++ out->end_list(out);
++
++ g_strfreev(feature_list);
++ return pcmk_rc_ok;
++}
++
++static pcmk__message_entry_t fmt_functions[] = {
++ { "features", "default", pacemakerd_features },
++ { "features", "xml", pacemakerd_features_xml },
++
+ { NULL, NULL, NULL }
+ };
+
+@@ -200,7 +236,7 @@ static GOptionContext *
+ build_arg_context(pcmk__common_args_t *args, GOptionGroup **group) {
+ GOptionContext *context = NULL;
+
+- context = pcmk__build_arg_context(args, "text", group, NULL);
++ context = pcmk__build_arg_context(args, "text (default), xml", group,
NULL);
+ pcmk__add_main_args(context, entries);
+ return context;
+ }
+@@ -241,9 +277,12 @@ main(int argc, char **argv)
+ goto done;
+ }
+
++ pcmk__force_args(context, &error, "%s --xml-simple-list",
g_get_prgname());
++
++ pcmk__register_messages(out, fmt_functions);
++
+ if (options.features) {
+- out->info(out, "Pacemaker %s (Build: %s)\n Supporting v%s: %s",
PACEMAKER_VERSION,
+- BUILD_VERSION, CRM_FEATURE_SET, CRM_FEATURES);
++ out->message(out, "features");
+ exit_code = CRM_EX_OK;
+ goto done;
+ }
+--
+1.8.3.1
+
+
+From 5b7f5eb35b025b59805cf3c7c3dcb6a3cf4b71b3 Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Mon, 17 May 2021 11:09:53 -0400
+Subject: [PATCH 3/5] Low: daemons: Conditionally enable logging in pacemakerd.
+
+If we're doing an interactive command-line call, use
+pcmk__cli_init_logging. At the moment, all command line calls except
+for --shutdown do their work before logging would even come up, so we
+really only need to do this for --shutdown.
+
+If we're doing a daemon call, use crm_log_init.
+---
+ daemons/pacemakerd/pacemakerd.c | 7 +++++--
+ 1 file changed, 5 insertions(+), 2 deletions(-)
+
+diff --git a/daemons/pacemakerd/pacemakerd.c b/daemons/pacemakerd/pacemakerd.c
+index 93cf743..c20bde7 100644
+--- a/daemons/pacemakerd/pacemakerd.c
++++ b/daemons/pacemakerd/pacemakerd.c
+@@ -296,8 +296,11 @@ main(int argc, char **argv)
+
+ pcmk__set_env_option("mcp", "true");
+
+- pcmk__cli_init_logging("pacemakerd", args->verbosity);
+- crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
++ if (options.shutdown) {
++ pcmk__cli_init_logging("pacemakerd", args->verbosity);
++ } else {
++ crm_log_init(NULL, LOG_INFO, TRUE, FALSE, argc, argv, FALSE);
++ }
+
+ crm_debug("Checking for existing Pacemaker instance");
+ old_instance = crm_ipc_new(CRM_SYSTEM_MCP, 0);
+--
+1.8.3.1
+
+
+From 2393362bb7489e86d937ed46a1c5cfb93d9bf3ab Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Mon, 17 May 2021 11:58:06 -0400
+Subject: [PATCH 4/5] Fix: include: Bump CRM_FEATURE_SET for new pacemakerd
+ args.
+
+---
+ include/crm/crm.h | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/include/crm/crm.h b/include/crm/crm.h
+index fdfc825..92a98fa 100644
+--- a/include/crm/crm.h
++++ b/include/crm/crm.h
+@@ -66,7 +66,7 @@ extern "C" {
+ * >=3.0.13: Fail counts include operation name and interval
+ * >=3.2.0: DC supports PCMK_LRM_OP_INVALID and PCMK_LRM_OP_NOT_CONNECTED
+ */
+-# define CRM_FEATURE_SET "3.10.0"
++# define CRM_FEATURE_SET "3.10.1"
+
+ /* Pacemaker's CPG protocols use fixed-width binary fields for the sender and
+ * recipient of a CPG message. This imposes an arbitrary limit on cluster node
+--
+1.8.3.1
+
+
+From 3ad8edbd91631b87ef5f53fa2d68f0c8bbb9ee2b Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Mon, 17 May 2021 11:57:09 -0400
+Subject: [PATCH 5/5] Feature: xml: Add schema for pacemakerd.
+
+---
+ xml/Makefile.am | 1 +
+ xml/api/pacemakerd-2.10.rng | 28 ++++++++++++++++++++++++++++
+ 2 files changed, 29 insertions(+)
+ create mode 100644 xml/api/pacemakerd-2.10.rng
+
+diff --git a/xml/Makefile.am b/xml/Makefile.am
+index 12a51c5..b9448d4 100644
+--- a/xml/Makefile.am
++++ b/xml/Makefile.am
+@@ -56,6 +56,7 @@ API_request_base = command-output \
+ crm_simulate \
+ crmadmin \
+ digests \
++ pacemakerd \
+ stonith_admin \
+ version
+
+diff --git a/xml/api/pacemakerd-2.10.rng b/xml/api/pacemakerd-2.10.rng
+new file mode 100644
+index 0000000..41a11e7
+--- /dev/null
++++ b/xml/api/pacemakerd-2.10.rng
+@@ -0,0 +1,28 @@
++<?xml version="1.0" encoding="UTF-8"?>
++<grammar
xmlns="http://relaxng.org/ns/structure/1.0"
++
datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
++
++ <start>
++ <ref name="element-pacemakerd"/>
++ </start>
++
++ <define name="element-pacemakerd">
++ <element name="pacemakerd">
++ <attribute name="version"> <text />
</attribute>
++ <attribute name="build"> <text /> </attribute>
++ <attribute name="feature_set"> <text />
</attribute>
++
++ <optional>
++ <ref name="feature-list" />
++ </optional>
++ </element>
++ </define>
++
++ <define name="feature-list">
++ <element name="features">
++ <oneOrMore>
++ <element name="feature"> <text/> </element>
++ </oneOrMore>
++ </element>
++ </define>
++</grammar>
+--
+1.8.3.1
+
diff --git a/004-check-level.patch b/004-check-level.patch
new file mode 100644
index 0000000..f2abb5f
--- /dev/null
+++ b/004-check-level.patch
@@ -0,0 +1,199 @@
+From 3905e7eac11298fc20efd567a773666f948edf61 Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Mon, 3 May 2021 11:19:04 -0400
+Subject: [PATCH 1/2] Feature: tools: Add OCF_CHECK_LEVEL to crm_resource
+ environment.
+
+If --validate= or --force-check= are given with a level, pass that along
+as OCF_CHECK_LEVEL. This argument is optional, and if no value is given
+then the environment variable will not be set and whatever's the default
+on the resource agent will be used.
+
+See: rhbz#1955792.
+---
+ tools/crm_resource.c | 29 +++++++++++++++++++++--------
+ tools/crm_resource.h | 4 ++--
+ tools/crm_resource_runtime.c | 13 ++++++++++---
+ 3 files changed, 33 insertions(+), 13 deletions(-)
+
+diff --git a/tools/crm_resource.c b/tools/crm_resource.c
+index 45db2b2..6ca96f8 100644
+--- a/tools/crm_resource.c
++++ b/tools/crm_resource.c
+@@ -100,6 +100,7 @@ struct {
+ int timeout_ms; // Parsed from --timeout value
+ char *agent_spec; // Standard and/or provider and/or agent
+ gchar *xml_file; // Value of (deprecated) --xml-file
++ int check_level; // Optional value of --validate or --force-check
+
+ // Resource configuration specified via command-line arguments
+ gboolean cmdline_config; // Resource configuration was via arguments
+@@ -113,6 +114,7 @@ struct {
+ GHashTable *override_params; // Resource parameter values that override config
+ } options = {
+ .attr_set_type = XML_TAG_ATTR_SETS,
++ .check_level = -1,
+ .cib_options = cib_sync_call,
+ .require_cib = TRUE,
+ .require_dataset = TRUE,
+@@ -402,14 +404,15 @@ static GOptionEntry query_entries[] = {
+ };
+
+ static GOptionEntry command_entries[] = {
+- { "validate", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
++ { "validate", 0, G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
+ validate_or_force_cb,
+ "Validate resource configuration by calling agent's validate-all\n"
+ INDENT "action. The configuration may be specified either by giving
an\n"
+ INDENT "existing resource name with -r, or by specifying --class,\n"
+ INDENT "--agent, and --provider arguments, along with any number of\n"
+- INDENT "--option arguments.",
+- NULL },
++ INDENT "--option arguments. An optional LEVEL argument can be given\n"
++ INDENT "to control the level of checking performed.",
++ "LEVEL" },
+ { "cleanup", 'C', G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
cleanup_refresh_cb,
+ "If resource has any past failures, clear its history and fail\n"
+ INDENT "count. Optionally filtered by --resource, --node,
--operation\n"
+@@ -546,11 +549,12 @@ static GOptionEntry advanced_entries[] = {
+ INDENT "the cluster believes the resource is a clone instance
already\n"
+ INDENT "running on the local node.",
+ NULL },
+- { "force-check", 0, G_OPTION_FLAG_NO_ARG, G_OPTION_ARG_CALLBACK,
++ { "force-check", 0, G_OPTION_FLAG_OPTIONAL_ARG, G_OPTION_ARG_CALLBACK,
+ validate_or_force_cb,
+ "(Advanced) Bypass the cluster and check the state of a resource on\n"
+- INDENT "the local node",
+- NULL },
++ INDENT "the local node. An optional LEVEL argument can be given\n"
++ INDENT "to control the level of checking performed.",
++ "LEVEL" },
+
+ { NULL }
+ };
+@@ -910,6 +914,15 @@ validate_or_force_cb(const gchar *option_name, const gchar *optarg,
+ if (options.override_params == NULL) {
+ options.override_params = pcmk__strkey_table(free, free);
+ }
++
++ if (optarg != NULL) {
++ if (pcmk__scan_min_int(optarg, &options.check_level, 0) != pcmk_rc_ok) {
++ g_set_error(error, G_OPTION_ERROR, CRM_EX_INVALID_PARAM,
++ "Invalid check level setting: %s", optarg);
++ return FALSE;
++ }
++ }
++
+ return TRUE;
+ }
+
+@@ -1826,12 +1839,12 @@ main(int argc, char **argv)
+ options.v_class, options.v_provider, options.v_agent,
+ "validate-all", options.cmdline_params,
+ options.override_params, options.timeout_ms,
+- args->verbosity, options.force);
++ args->verbosity, options.force, options.check_level);
+ } else {
+ exit_code = cli_resource_execute(rsc, options.rsc_id,
+ options.operation, options.override_params,
+ options.timeout_ms, cib_conn, data_set,
+- args->verbosity, options.force);
++ args->verbosity, options.force, options.check_level);
+ }
+ goto done;
+
+diff --git a/tools/crm_resource.h b/tools/crm_resource.h
+index 3560377..5ab10d6 100644
+--- a/tools/crm_resource.h
++++ b/tools/crm_resource.h
+@@ -88,11 +88,11 @@ crm_exit_t cli_resource_execute_from_params(pcmk__output_t *out,
const char *rsc
+ const char *rsc_type, const char
*rsc_action,
+ GHashTable *params, GHashTable
*override_hash,
+ int timeout_ms, int resource_verbose,
+- gboolean force);
++ gboolean force, int check_level);
+ crm_exit_t cli_resource_execute(pe_resource_t *rsc, const char *requested_name,
+ const char *rsc_action, GHashTable *override_hash,
+ int timeout_ms, cib_t *cib, pe_working_set_t *data_set,
+- int resource_verbose, gboolean force);
++ int resource_verbose, gboolean force, int check_level);
+
+ int cli_resource_update_attribute(pe_resource_t *rsc, const char *requested_name,
+ const char *attr_set, const char *attr_set_type,
+diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
+index fe0ec98..bde83b6 100644
+--- a/tools/crm_resource_runtime.c
++++ b/tools/crm_resource_runtime.c
+@@ -1679,7 +1679,8 @@ cli_resource_execute_from_params(pcmk__output_t *out, const char
*rsc_name,
+ const char *rsc_class, const char *rsc_prov,
+ const char *rsc_type, const char *action,
+ GHashTable *params, GHashTable *override_hash,
+- int timeout_ms, int resource_verbose, gboolean force)
++ int timeout_ms, int resource_verbose, gboolean force,
++ int check_level)
+ {
+ GHashTable *params_copy = NULL;
+ crm_exit_t exit_code = CRM_EX_OK;
+@@ -1703,6 +1704,12 @@ cli_resource_execute_from_params(pcmk__output_t *out, const char
*rsc_name,
+ /* add crm_feature_set env needed by some resource agents */
+ g_hash_table_insert(params, strdup(XML_ATTR_CRM_VERSION), strdup(CRM_FEATURE_SET));
+
++ if (check_level >= 0) {
++ char *level = crm_strdup_printf("%d", check_level);
++ setenv("OCF_CHECK_LEVEL", level, 1);
++ free(level);
++ }
++
+ /* resources_action_create frees the params hash table it's passed, but we
+ * may need to reuse it in a second call to resources_action_create. Thus
+ * we'll make a copy here so that gets freed and the original remains for
+@@ -1790,7 +1797,7 @@ crm_exit_t
+ cli_resource_execute(pe_resource_t *rsc, const char *requested_name,
+ const char *rsc_action, GHashTable *override_hash,
+ int timeout_ms, cib_t * cib, pe_working_set_t *data_set,
+- int resource_verbose, gboolean force)
++ int resource_verbose, gboolean force, int check_level)
+ {
+ pcmk__output_t *out = data_set->priv;
+ crm_exit_t exit_code = CRM_EX_OK;
+@@ -1856,7 +1863,7 @@ cli_resource_execute(pe_resource_t *rsc, const char
*requested_name,
+
+ exit_code = cli_resource_execute_from_params(out, rid, rclass, rprov, rtype,
action,
+ params, override_hash, timeout_ms,
+- resource_verbose, force);
++ resource_verbose, force, check_level);
+ return exit_code;
+ }
+
+--
+1.8.3.1
+
+
+From d13ba4bd6defe0dd81fdf8ab39ae5b889513c0c0 Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Thu, 20 May 2021 10:59:23 -0400
+Subject: [PATCH 2/2] Fix: include: Bump feature set to 3.10.2.
+
+This is for the OCF_CHECK_LEVEL environment variable.
+
+See: rhbz#1955792.
+---
+ include/crm/crm.h | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/include/crm/crm.h b/include/crm/crm.h
+index 92a98fa..ee52c36 100644
+--- a/include/crm/crm.h
++++ b/include/crm/crm.h
+@@ -66,7 +66,7 @@ extern "C" {
+ * >=3.0.13: Fail counts include operation name and interval
+ * >=3.2.0: DC supports PCMK_LRM_OP_INVALID and PCMK_LRM_OP_NOT_CONNECTED
+ */
+-# define CRM_FEATURE_SET "3.10.1"
++# define CRM_FEATURE_SET "3.10.2"
+
+ /* Pacemaker's CPG protocols use fixed-width binary fields for the sender and
+ * recipient of a CPG message. This imposes an arbitrary limit on cluster node
+--
+1.8.3.1
+
diff --git a/005-crm_resource.patch b/005-crm_resource.patch
new file mode 100644
index 0000000..1683026
--- /dev/null
+++ b/005-crm_resource.patch
@@ -0,0 +1,866 @@
+From a5a507d4e1abf242903472719a19977811e6f164 Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Thu, 20 May 2021 11:59:36 -0400
+Subject: [PATCH 01/10] Feature: libcrmcommon: Add OCF_OUTPUT_FORMAT to
+ crm_resource environment.
+
+See: rhbz#1644628
+---
+ lib/common/output.c | 2 ++
+ 1 file changed, 2 insertions(+)
+
+diff --git a/lib/common/output.c b/lib/common/output.c
+index 6cb49b5..58872e0 100644
+--- a/lib/common/output.c
++++ b/lib/common/output.c
+@@ -71,6 +71,8 @@ pcmk__output_new(pcmk__output_t **out, const char *fmt_name, const char
*filenam
+ return ENOMEM;
+ }
+
++ setenv("OCF_OUTPUT_FORMAT", (*out)->fmt_name, 1);
++
+ return pcmk_rc_ok;
+ }
+
+--
+1.8.3.1
+
+
+From acc6ecdbfb797d69794e68f75a734d6252434e01 Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Fri, 21 May 2021 14:20:30 -0400
+Subject: [PATCH 02/10] Feature: schemas: Copy crm_resource schema in
+ preparation for changes.
+
+See: rhbz#1644628
+---
+ xml/api/crm_resource-2.11.rng | 238 ++++++++++++++++++++++++++++++++++++++++++
+ 1 file changed, 238 insertions(+)
+ create mode 100644 xml/api/crm_resource-2.11.rng
+
+diff --git a/xml/api/crm_resource-2.11.rng b/xml/api/crm_resource-2.11.rng
+new file mode 100644
+index 0000000..8e386db
+--- /dev/null
++++ b/xml/api/crm_resource-2.11.rng
+@@ -0,0 +1,238 @@
++<?xml version="1.0" encoding="UTF-8"?>
++<grammar
xmlns="http://relaxng.org/ns/structure/1.0"
++
datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
++
++ <start>
++ <ref name="element-crm-resource"/>
++ </start>
++
++ <define name="element-crm-resource">
++ <choice>
++ <ref name="agents-list" />
++ <ref name="alternatives-list" />
++ <ref name="constraints-list" />
++ <externalRef href="generic-list-2.4.rng"/>
++ <element name="metadata"> <text/> </element>
++ <ref name="locate-list" />
++ <ref name="operations-list" />
++ <ref name="providers-list" />
++ <ref name="reasons-list" />
++ <ref name="resource-check" />
++ <ref name="resource-config" />
++ <ref name="resources-list" />
++ </choice>
++ </define>
++
++ <define name="agents-list">
++ <element name="agents">
++ <attribute name="standard"> <text/>
</attribute>
++ <optional>
++ <attribute name="provider"> <text/>
</attribute>
++ </optional>
++ <zeroOrMore>
++ <element name="agent"> <text/> </element>
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="alternatives-list">
++ <element name="providers">
++ <attribute name="for"> <text/> </attribute>
++ <zeroOrMore>
++ <element name="provider"> <text/>
</element>
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="constraints-list">
++ <element name="constraints">
++ <interleave>
++ <zeroOrMore>
++ <ref name="rsc-location" />
++ </zeroOrMore>
++ <zeroOrMore>
++ <ref name="rsc-colocation" />
++ </zeroOrMore>
++ </interleave>
++ </element>
++ </define>
++
++ <define name="locate-list">
++ <element name="nodes">
++ <attribute name="resource"> <text/>
</attribute>
++ <zeroOrMore>
++ <element name="node">
++ <optional>
++ <attribute
name="state"><value>promoted</value></attribute>
++ </optional>
++ <text/>
++ </element>
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="rsc-location">
++ <element name="rsc_location">
++ <attribute name="node"> <text/> </attribute>
++ <attribute name="rsc"> <text/> </attribute>
++ <attribute name="id"> <text/> </attribute>
++ <externalRef href="../score.rng"/>
++ </element>
++ </define>
++
++ <define name="operations-list">
++ <element name="operations">
++ <oneOrMore>
++ <ref name="element-operation-list" />
++ </oneOrMore>
++ </element>
++ </define>
++
++ <define name="providers-list">
++ <element name="providers">
++ <attribute name="standard"> <value>ocf</value>
</attribute>
++ <optional>
++ <attribute name="agent"> <text/>
</attribute>
++ </optional>
++ <zeroOrMore>
++ <element name="provider"> <text/>
</element>
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="reasons-list">
++ <choice>
++ <ref name="no-resource-or-uname"/>
++ <ref name="resource-and-uname"/>
++ <ref name="no-resource-but-uname"/>
++ <ref name="resource-but-no-uname"/>
++ </choice>
++ </define>
++
++ <define name="no-resource-or-uname">
++ <element name="reason">
++ <element name="resources">
++ <zeroOrMore>
++ <element name="resource">
++ <attribute name="id"> <text/>
</attribute>
++ <attribute name="running"> <data
type="boolean"/> </attribute>
++ <ref name="resource-check"/>
++ </element>
++ </zeroOrMore>
++ </element>
++ </element>
++ </define>
++
++ <define name="resource-and-uname">
++ <element name="reason">
++ <attribute name="running_on"> <text/>
</attribute>
++ <ref name="resource-check"/>
++ </element>
++ </define>
++
++ <define name="no-resource-but-uname">
++ <element name="reason">
++ <element name="resources">
++ <zeroOrMore>
++ <element name="resource">
++ <attribute name="id"> <text/>
</attribute>
++ <attribute name="running"> <data
type="boolean"/> </attribute>
++ <attribute name="host"> <text/>
</attribute>
++ <ref name="resource-check"/>
++ </element>
++ </zeroOrMore>
++ </element>
++ </element>
++ </define>
++
++ <define name="resource-but-no-uname">
++ <element name="reason">
++ <attribute name="running"> <data
type="boolean"/> </attribute>
++ <ref name="resource-check"/>
++ </element>
++ </define>
++
++ <define name="resource-config">
++ <element name="resource_config">
++ <externalRef href="resources-2.4.rng" />
++ <element name="xml"> <text/> </element>
++ </element>
++ </define>
++
++ <define name="resource-check">
++ <element name="check">
++ <attribute name="id"> <text/> </attribute>
++ <optional>
++ <choice>
++ <attribute
name="remain_stopped"><value>true</value></attribute>
++ <attribute
name="promotable"><value>false</value></attribute>
++ </choice>
++ </optional>
++ <optional>
++ <attribute
name="unmanaged"><value>true</value></attribute>
++ </optional>
++ <optional>
++ <attribute name="locked-to"> <text/>
</attribute>
++ </optional>
++ </element>
++ </define>
++
++ <define name="resources-list">
++ <element name="resources">
++ <zeroOrMore>
++ <externalRef href="resources-2.4.rng" />
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="rsc-colocation">
++ <element name="rsc_colocation">
++ <attribute name="id"> <text/> </attribute>
++ <attribute name="rsc"> <text/> </attribute>
++ <attribute name="with-rsc"> <text/>
</attribute>
++ <externalRef href="../score.rng"/>
++ <optional>
++ <attribute name="node-attribute"> <text/>
</attribute>
++ </optional>
++ <optional>
++ <attribute name="rsc-role">
++ <ref name="attribute-roles"/>
++ </attribute>
++ </optional>
++ <optional>
++ <attribute name="with-rsc-role">
++ <ref name="attribute-roles"/>
++ </attribute>
++ </optional>
++ </element>
++ </define>
++
++ <define name="element-operation-list">
++ <element name="operation">
++ <optional>
++ <group>
++ <attribute name="rsc"> <text/>
</attribute>
++ <attribute name="agent"> <text/>
</attribute>
++ </group>
++ </optional>
++ <attribute name="op"> <text/> </attribute>
++ <attribute name="node"> <text/> </attribute>
++ <attribute name="call"> <data type="integer"
/> </attribute>
++ <attribute name="rc"> <data
type="nonNegativeInteger" /> </attribute>
++ <optional>
++ <attribute name="last-rc-change"> <text/>
</attribute>
++ <attribute name="exec-time"> <data
type="nonNegativeInteger" /> </attribute>
++ </optional>
++ <attribute name="status"> <text/> </attribute>
++ </element>
++ </define>
++
++ <define name="attribute-roles">
++ <choice>
++ <value>Stopped</value>
++ <value>Started</value>
++ <value>Master</value>
++ <value>Slave</value>
++ </choice>
++ </define>
++</grammar>
+--
+1.8.3.1
+
+
+From 1bbdf2149a111e9e19c388834f82001e0d31c427 Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Mon, 24 May 2021 12:23:55 -0400
+Subject: [PATCH 03/10] Feature: xml: Update the crm_resource schema for XML
+ output.
+
+See: rhbz#1644628
+---
+ xml/api/crm_resource-2.11.rng | 50 +++++++++++++++++++++++++++++++++++++++++++
+ 1 file changed, 50 insertions(+)
+
+diff --git a/xml/api/crm_resource-2.11.rng b/xml/api/crm_resource-2.11.rng
+index 8e386db..aaa54d6 100644
+--- a/xml/api/crm_resource-2.11.rng
++++ b/xml/api/crm_resource-2.11.rng
+@@ -20,6 +20,7 @@
+ <ref name="resource-check" />
+ <ref name="resource-config" />
+ <ref name="resources-list" />
++ <ref name="resource-agent-action" />
+ </choice>
+ </define>
+
+@@ -227,6 +228,55 @@
+ </element>
+ </define>
+
++ <define name="resource-agent-action">
++ <element name="resource-agent-action">
++ <attribute name="action"> <text/> </attribute>
++ <optional>
++ <attribute name="rsc"> <text/> </attribute>
++ </optional>
++ <attribute name="class"> <text/> </attribute>
++ <attribute name="type"> <text/> </attribute>
++ <optional>
++ <attribute name="provider"> <text/>
</attribute>
++ </optional>
++ <optional>
++ <ref name="overrides-list"/>
++ </optional>
++ <ref name="agent-status"/>
++ <optional>
++ <choice>
++ <element name="command">
++ <text />
++ </element>
++ <externalRef href="command-output-1.0.rng"/>
++ </choice>
++ </optional>
++ </element>
++ </define>
++
++ <define name="overrides-list">
++ <element name="overrides">
++ <zeroOrMore>
++ <element name="override">
++ <optional>
++ <attribute name="rsc"> <text/>
</attribute>
++ </optional>
++ <attribute name="name"> <text/>
</attribute>
++ <attribute name="value"> <text/>
</attribute>
++ </element>
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="agent-status">
++ <element name="agent-status">
++ <attribute name="code"> <data type="integer"
/> </attribute>
++ <optional>
++ <attribute name="message"> <text/>
</attribute>
++ </optional>
++ </element>
++ </define>
++
+ <define name="attribute-roles">
+ <choice>
+ <value>Stopped</value>
+--
+1.8.3.1
+
+
+From d89f5bc7fec856fdcd32fa14edbd0019507d5d15 Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Tue, 1 Jun 2021 15:26:58 -0400
+Subject: [PATCH 04/10] Low: libcrmcommon: Increase PCMK__API_VERSION for new
+ crm_resource output.
+
+See: rhbz#1644628
+---
+ include/crm/common/output_internal.h | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/include/crm/common/output_internal.h b/include/crm/common/output_internal.h
+index 10b315b..0436cde 100644
+--- a/include/crm/common/output_internal.h
++++ b/include/crm/common/output_internal.h
+@@ -27,7 +27,7 @@ extern "C" {
+ # include <glib.h>
+ # include <crm/common/results.h>
+
+-# define PCMK__API_VERSION "2.9"
++# define PCMK__API_VERSION "2.11"
+
+ #if defined(PCMK__WITH_ATTRIBUTE_OUTPUT_ARGS)
+ # define PCMK__OUTPUT_ARGS(ARGS...) __attribute__((output_args(ARGS)))
+--
+1.8.3.1
+
+
+From 30bd2ddf43ee2a911681e51f40ed9ba20ec250b0 Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Thu, 27 May 2021 13:57:12 -0400
+Subject: [PATCH 05/10] Low: tools: Pass NULL to
+ cli_resource_execute_from_params...
+
+if no resource name is given. This happens if we are validating based
+on the --class/--agent/--provider command line options instead.
+---
+ tools/crm_resource.c | 2 +-
+ tools/crm_resource_runtime.c | 8 ++++----
+ 2 files changed, 5 insertions(+), 5 deletions(-)
+
+diff --git a/tools/crm_resource.c b/tools/crm_resource.c
+index 24f1121..37a0bb0 100644
+--- a/tools/crm_resource.c
++++ b/tools/crm_resource.c
+@@ -1840,7 +1840,7 @@ main(int argc, char **argv)
+
+ case cmd_execute_agent:
+ if (options.cmdline_config) {
+- exit_code = cli_resource_execute_from_params(out, "test",
++ exit_code = cli_resource_execute_from_params(out, NULL,
+ options.v_class, options.v_provider, options.v_agent,
+ "validate-all", options.cmdline_params,
+ options.override_params, options.timeout_ms,
+diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
+index 48a4b40..ebf48bb 100644
+--- a/tools/crm_resource_runtime.c
++++ b/tools/crm_resource_runtime.c
+@@ -1717,14 +1717,14 @@ cli_resource_execute_from_params(pcmk__output_t *out, const char
*rsc_name,
+ */
+ params_copy = pcmk__str_table_dup(params);
+
+- op = resources_action_create(rsc_name, rsc_class, rsc_prov, rsc_type, action, 0,
+- timeout_ms, params_copy, 0);
++ op = resources_action_create(rsc_name ? rsc_name : "test", rsc_class,
rsc_prov,
++ rsc_type, action, 0, timeout_ms, params_copy, 0);
+ if (op == NULL) {
+ /* Re-run with stderr enabled so we can display a sane error message */
+ crm_enable_stderr(TRUE);
+ params_copy = pcmk__str_table_dup(params);
+- op = resources_action_create(rsc_name, rsc_class, rsc_prov, rsc_type, action,
0,
+- timeout_ms, params_copy, 0);
++ op = resources_action_create(rsc_name ? rsc_name : "test", rsc_class,
rsc_prov,
++ rsc_type, action, 0, timeout_ms, params_copy, 0);
+
+ /* Callers of cli_resource_execute expect that the params hash table will
+ * be freed. That function uses this one, so for that reason and for
+--
+1.8.3.1
+
+
+From ee56efd53d14cfc4f902769540b72b3bb6096a73 Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Mon, 24 May 2021 12:08:52 -0400
+Subject: [PATCH 06/10] Feature: tools: Add an agent-status message for
+ crm_resource.
+
+This moves what was previously only done in an out->info call to its own
+output message, which means it will appear in XML output as well. Also,
+note that if --class/--agent/--provider are given, the resource name
+will be set to "test". In that case, do not display the resource name
+in the output.
+
+This message will be used for --validate and the --force-* command line
+options to crm_resource.
+
+See: rhbz#1644628
+---
+ tools/crm_resource_print.c | 53 ++++++++++++++++++++++++++++++++++++++++++++++
+ 1 file changed, 53 insertions(+)
+
+diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
+index 9d82cf8..88d5878 100644
+--- a/tools/crm_resource_print.c
++++ b/tools/crm_resource_print.c
+@@ -152,6 +152,57 @@ attribute_list_default(pcmk__output_t *out, va_list args) {
+ return pcmk_rc_ok;
+ }
+
++PCMK__OUTPUT_ARGS("agent-status", "int", "const char *",
"const char *", "const char *",
++ "const char *", "const char *", "int")
++static int
++agent_status_default(pcmk__output_t *out, va_list args) {
++ int status = va_arg(args, int);
++ const char *action = va_arg(args, const char *);
++ const char *name = va_arg(args, const char *);
++ const char *class = va_arg(args, const char *);
++ const char *provider = va_arg(args, const char *);
++ const char *type = va_arg(args, const char *);
++ int rc = va_arg(args, int);
++
++ if (status == PCMK_LRM_OP_DONE) {
++ out->info(out, "Operation %s%s%s (%s%s%s:%s) returned: '%s'
(%d)",
++ action, name ? " for " : "", name ? name :
"",
++ class, provider ? ":" : "", provider ? provider :
"", type,
++ services_ocf_exitcode_str(rc), rc);
++ } else {
++ out->err(out, "Operation %s%s%s (%s%s%s:%s) failed: '%s'
(%d)",
++ action, name ? " for " : "", name ? name :
"",
++ class, provider ? ":" : "", provider ? provider :
"", type,
++ services_lrm_status_str(status), status);
++ }
++
++ return pcmk_rc_ok;
++}
++
++PCMK__OUTPUT_ARGS("agent-status", "int", "const char *",
"const char *", "const char *",
++ "const char *", "const char *", "int")
++static int
++agent_status_xml(pcmk__output_t *out, va_list args) {
++ int status G_GNUC_UNUSED = va_arg(args, int);
++ const char *action G_GNUC_UNUSED = va_arg(args, const char *);
++ const char *name G_GNUC_UNUSED = va_arg(args, const char *);
++ const char *class G_GNUC_UNUSED = va_arg(args, const char *);
++ const char *provider G_GNUC_UNUSED = va_arg(args, const char *);
++ const char *type G_GNUC_UNUSED = va_arg(args, const char *);
++ int rc = va_arg(args, int);
++
++ char *status_str = pcmk__itoa(rc);
++
++ pcmk__output_create_xml_node(out, "agent-status",
++ "code", status_str,
++ "message", services_ocf_exitcode_str(rc),
++ NULL);
++
++ free(status_str);
++
++ return pcmk_rc_ok;
++}
++
+ PCMK__OUTPUT_ARGS("attribute-list", "pe_resource_t *", "char
*", "GHashTable *")
+ static int
+ attribute_list_text(pcmk__output_t *out, va_list args) {
+@@ -562,6 +613,8 @@ resource_names(pcmk__output_t *out, va_list args) {
+ }
+
+ static pcmk__message_entry_t fmt_functions[] = {
++ { "agent-status", "default", agent_status_default },
++ { "agent-status", "xml", agent_status_xml },
+ { "attribute-list", "default", attribute_list_default },
+ { "attribute-list", "text", attribute_list_text },
+ { "property-list", "default", property_list_default },
+--
+1.8.3.1
+
+
+From 85cb6b6bff96b18c5174d11e4de4d49cbfb20bb7 Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Tue, 1 Jun 2021 14:47:30 -0400
+Subject: [PATCH 07/10] Feature: tools: Add an overridden params output
+ message.
+
+This also replaces what was previously being done in an out->info call
+with an output message. This means it shows up in XML output as well.
+Also, note that if --class/--agent/--provider are given, the resource
+name will be set to "test". In that case, do not display the resource
+name in the output.
+
+See: rhbz#1644628
+---
+ tools/crm_resource_print.c | 39 +++++++++++++++++++++++++++++++++++++++
+ 1 file changed, 39 insertions(+)
+
+diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
+index 88d5878..119d83f 100644
+--- a/tools/crm_resource_print.c
++++ b/tools/crm_resource_print.c
+@@ -224,6 +224,43 @@ attribute_list_text(pcmk__output_t *out, va_list args) {
+ return pcmk_rc_ok;
+ }
+
++PCMK__OUTPUT_ARGS("override", "const char *", "const char
*", "const char *")
++static int
++override_default(pcmk__output_t *out, va_list args) {
++ const char *rsc_name = va_arg(args, const char *);
++ const char *name = va_arg(args, const char *);
++ const char *value = va_arg(args, const char *);
++
++ if (rsc_name == NULL) {
++ out->list_item(out, NULL, "Overriding the cluster configuration with
'%s' = '%s'",
++ name, value);
++ } else {
++ out->list_item(out, NULL, "Overriding the cluster configuration for
'%s' with '%s' = '%s'",
++ rsc_name, name, value);
++ }
++
++ return pcmk_rc_ok;
++}
++
++PCMK__OUTPUT_ARGS("override", "const char *", "const char
*", "const char *")
++static int
++override_xml(pcmk__output_t *out, va_list args) {
++ const char *rsc_name = va_arg(args, const char *);
++ const char *name = va_arg(args, const char *);
++ const char *value = va_arg(args, const char *);
++
++ xmlNodePtr node = pcmk__output_create_xml_node(out, "override",
++ "name", name,
++ "value", value,
++ NULL);
++
++ if (rsc_name != NULL) {
++ crm_xml_add(node, "rsc", rsc_name);
++ }
++
++ return pcmk_rc_ok;
++}
++
+ PCMK__OUTPUT_ARGS("property-list", "pe_resource_t *", "char
*")
+ static int
+ property_list_default(pcmk__output_t *out, va_list args) {
+@@ -617,6 +654,8 @@ static pcmk__message_entry_t fmt_functions[] = {
+ { "agent-status", "xml", agent_status_xml },
+ { "attribute-list", "default", attribute_list_default },
+ { "attribute-list", "text", attribute_list_text },
++ { "override", "default", override_default },
++ { "override", "xml", override_xml },
+ { "property-list", "default", property_list_default },
+ { "property-list", "text", property_list_text },
+ { "resource-check-list", "default", resource_check_list_default
},
+--
+1.8.3.1
+
+
+From e5e24592c7c3231c619fb5253e7925ffbc634a99 Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Fri, 4 Jun 2021 10:24:51 -0400
+Subject: [PATCH 08/10] Low: tools: Use simple XML lists for resource actions
+ as well.
+
+See: rhbz#1644628
+---
+ tools/crm_resource.c | 1 +
+ 1 file changed, 1 insertion(+)
+
+diff --git a/tools/crm_resource.c b/tools/crm_resource.c
+index 37a0bb0..e957011 100644
+--- a/tools/crm_resource.c
++++ b/tools/crm_resource.c
+@@ -1643,6 +1643,7 @@ main(int argc, char **argv)
+ * saves from having to write custom messages to build the lists around all
these things
+ */
+ switch (options.rsc_cmd) {
++ case cmd_execute_agent:
+ case cmd_list_resources:
+ case cmd_query_xml:
+ case cmd_query_raw_xml:
+--
+1.8.3.1
+
+
+From 3e75174d0bc31b261adb1994214a5878b79da85b Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Fri, 4 Jun 2021 10:30:10 -0400
+Subject: [PATCH 09/10] Feature: tools: Add an output message for resource
+ actions.
+
+This wraps up the override and agent-status messages into a single
+message, along with any stdout/stderr from the resource action. This
+message should be called after taking the action.
+
+This also implements handling XML output from resource actions. Check
+to see if the validate-all action returns XML. If so, output it as a
+CDATA block under a "command" element. If not, treat it as plain text
+and output it as stdout/stderr from a command.
+
+See: rhbz#1644628
+---
+ tools/crm_resource_print.c | 122 +++++++++++++++++++++++++++++++++++++++++++++
+ 1 file changed, 122 insertions(+)
+
+diff --git a/tools/crm_resource_print.c b/tools/crm_resource_print.c
+index 119d83f..19a366d 100644
+--- a/tools/crm_resource_print.c
++++ b/tools/crm_resource_print.c
+@@ -293,6 +293,126 @@ property_list_text(pcmk__output_t *out, va_list args) {
+ return pcmk_rc_ok;
+ }
+
++PCMK__OUTPUT_ARGS("resource-agent-action", "int", "const char
*", "const char *",
++ "const char *", "const char *", "const char
*", "GHashTable *",
++ "int", "int", "char *", "char
*")
++static int
++resource_agent_action_default(pcmk__output_t *out, va_list args) {
++ int verbose = va_arg(args, int);
++
++ const char *class = va_arg(args, const char *);
++ const char *provider = va_arg(args, const char *);
++ const char *type = va_arg(args, const char *);
++ const char *rsc_name = va_arg(args, const char *);
++ const char *action = va_arg(args, const char *);
++ GHashTable *overrides = va_arg(args, GHashTable *);
++ int rc = va_arg(args, int);
++ int status = va_arg(args, int);
++ char *stdout_data = va_arg(args, char *);
++ char *stderr_data = va_arg(args, char *);
++
++ if (overrides) {
++ GHashTableIter iter;
++ char *name = NULL;
++ char *value = NULL;
++
++ out->begin_list(out, NULL, NULL, "overrides");
++
++ g_hash_table_iter_init(&iter, overrides);
++ while (g_hash_table_iter_next(&iter, (gpointer *) &name, (gpointer *)
&value)) {
++ out->message(out, "override", rsc_name, name, value);
++ }
++
++ out->end_list(out);
++ }
++
++ out->message(out, "agent-status", status, action, rsc_name, class,
provider,
++ type, rc);
++
++ /* hide output for validate-all if not in verbose */
++ if (verbose == 0 && pcmk__str_eq(action, "validate-all",
pcmk__str_casei)) {
++ return pcmk_rc_ok;
++ }
++
++ if (stdout_data || stderr_data) {
++ xmlNodePtr doc = string2xml(stdout_data);
++
++ if (doc != NULL) {
++ out->output_xml(out, "command", stdout_data);
++ xmlFreeNode(doc);
++ } else {
++ out->subprocess_output(out, rc, stdout_data, stderr_data);
++ }
++ }
++
++ return pcmk_rc_ok;
++}
++
++PCMK__OUTPUT_ARGS("resource-agent-action", "int", "const char
*", "const char *",
++ "const char *", "const char *", "const char
*", "GHashTable *",
++ "int", "int", "char *", "char
*")
++static int
++resource_agent_action_xml(pcmk__output_t *out, va_list args) {
++ int verbose G_GNUC_UNUSED = va_arg(args, int);
++
++ const char *class = va_arg(args, const char *);
++ const char *provider = va_arg(args, const char *);
++ const char *type = va_arg(args, const char *);
++ const char *rsc_name = va_arg(args, const char *);
++ const char *action = va_arg(args, const char *);
++ GHashTable *overrides = va_arg(args, GHashTable *);
++ int rc = va_arg(args, int);
++ int status = va_arg(args, int);
++ char *stdout_data = va_arg(args, char *);
++ char *stderr_data = va_arg(args, char *);
++
++ xmlNodePtr node = pcmk__output_xml_create_parent(out,
"resource-agent-action",
++ "action", action,
++ "class", class,
++ "type", type,
++ NULL);
++
++ if (rsc_name) {
++ crm_xml_add(node, "rsc", rsc_name);
++ }
++
++ if (provider) {
++ crm_xml_add(node, "provider", provider);
++ }
++
++ if (overrides) {
++ GHashTableIter iter;
++ char *name = NULL;
++ char *value = NULL;
++
++ out->begin_list(out, NULL, NULL, "overrides");
++
++ g_hash_table_iter_init(&iter, overrides);
++ while (g_hash_table_iter_next(&iter, (gpointer *) &name, (gpointer *)
&value)) {
++ out->message(out, "override", rsc_name, name, value);
++ }
++
++ out->end_list(out);
++ }
++
++ out->message(out, "agent-status", status, action, rsc_name, class,
provider,
++ type, rc);
++
++ if (stdout_data || stderr_data) {
++ xmlNodePtr doc = string2xml(stdout_data);
++
++ if (doc != NULL) {
++ out->output_xml(out, "command", stdout_data);
++ xmlFreeNode(doc);
++ } else {
++ out->subprocess_output(out, rc, stdout_data, stderr_data);
++ }
++ }
++
++ pcmk__output_xml_pop_parent(out);
++ return pcmk_rc_ok;
++}
++
+ PCMK__OUTPUT_ARGS("resource-check-list", "resource_checks_t *")
+ static int
+ resource_check_list_default(pcmk__output_t *out, va_list args) {
+@@ -658,6 +778,8 @@ static pcmk__message_entry_t fmt_functions[] = {
+ { "override", "xml", override_xml },
+ { "property-list", "default", property_list_default },
+ { "property-list", "text", property_list_text },
++ { "resource-agent-action", "default",
resource_agent_action_default },
++ { "resource-agent-action", "xml", resource_agent_action_xml },
+ { "resource-check-list", "default", resource_check_list_default
},
+ { "resource-check-list", "xml", resource_check_list_xml },
+ { "resource-search-list", "default",
resource_search_list_default },
+--
+1.8.3.1
+
+
+From b50b2418e1e997b42f5370b4672a3f105d74634f Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Fri, 4 Jun 2021 10:40:16 -0400
+Subject: [PATCH 10/10] Feature: tools: Use the new resource-agent-action
+ message.
+
+See: rhbz#1644628
+---
+ tools/crm_resource_runtime.c | 21 +++------------------
+ 1 file changed, 3 insertions(+), 18 deletions(-)
+
+diff --git a/tools/crm_resource_runtime.c b/tools/crm_resource_runtime.c
+index ebf48bb..755be9f 100644
+--- a/tools/crm_resource_runtime.c
++++ b/tools/crm_resource_runtime.c
+@@ -1765,28 +1765,13 @@ cli_resource_execute_from_params(pcmk__output_t *out, const char
*rsc_name,
+ if (services_action_sync(op)) {
+ exit_code = op->rc;
+
+- if (op->status == PCMK_LRM_OP_DONE) {
+- out->info(out, "Operation %s for %s (%s:%s:%s) returned:
'%s' (%d)",
+- action, rsc_name, rsc_class, rsc_prov ? rsc_prov : "",
rsc_type,
+- services_ocf_exitcode_str(op->rc), op->rc);
+- } else {
+- out->err(out, "Operation %s for %s (%s:%s:%s) failed: '%s'
(%d)",
+- action, rsc_name, rsc_class, rsc_prov ? rsc_prov : "",
rsc_type,
+- services_lrm_status_str(op->status), op->status);
+- }
+-
+- /* hide output for validate-all if not in verbose */
+- if (resource_verbose == 0 && pcmk__str_eq(action,
"validate-all", pcmk__str_casei))
+- goto done;
+-
+- if (op->stdout_data || op->stderr_data) {
+- out->subprocess_output(out, op->rc, op->stdout_data,
op->stderr_data);
+- }
++ out->message(out, "resource-agent-action", resource_verbose,
rsc_class,
++ rsc_prov, rsc_type, rsc_name, action, override_hash, op->rc,
++ op->status, op->stdout_data, op->stderr_data);
+ } else {
+ exit_code = op->rc == 0 ? CRM_EX_ERROR : op->rc;
+ }
+
+-done:
+ services_action_free(op);
+ /* See comment above about why we free params here. */
+ g_hash_table_destroy(params);
+--
+1.8.3.1
+
diff --git a/006-crm_simulate.patch b/006-crm_simulate.patch
new file mode 100644
index 0000000..c8d4e3f
--- /dev/null
+++ b/006-crm_simulate.patch
@@ -0,0 +1,896 @@
+From 97571e6ccc9b7fa339a7e27d9b0b9ab782ff3003 Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Wed, 16 Jun 2021 13:54:10 -0400
+Subject: [PATCH 1/5] Low: schemas: Copy crm_mon.rng in preparation for
+ changes.
+
+---
+ xml/api/crm_mon-2.12.rng | 243 +++++++++++++++++++++++++++++++++++++++++++++++
+ 1 file changed, 243 insertions(+)
+ create mode 100644 xml/api/crm_mon-2.12.rng
+
+diff --git a/xml/api/crm_mon-2.12.rng b/xml/api/crm_mon-2.12.rng
+new file mode 100644
+index 0000000..ffec923
+--- /dev/null
++++ b/xml/api/crm_mon-2.12.rng
+@@ -0,0 +1,243 @@
++<?xml version="1.0" encoding="UTF-8"?>
++<grammar
xmlns="http://relaxng.org/ns/structure/1.0"
++
datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
++
++ <start>
++ <ref name="element-crm-mon"/>
++ </start>
++
++ <define name="element-crm-mon">
++ <optional>
++ <ref name="element-summary" />
++ </optional>
++ <optional>
++ <ref name="nodes-list" />
++ </optional>
++ <optional>
++ <ref name="resources-list" />
++ </optional>
++ <optional>
++ <ref name="node-attributes-list" />
++ </optional>
++ <optional>
++ <ref name="node-history-list" />
++ </optional>
++ <optional>
++ <ref name="failures-list" />
++ </optional>
++ <optional>
++ <ref name="fence-event-list" />
++ </optional>
++ <optional>
++ <ref name="tickets-list" />
++ </optional>
++ <optional>
++ <ref name="bans-list" />
++ </optional>
++ </define>
++
++ <define name="element-summary">
++ <element name="summary">
++ <optional>
++ <element name="stack">
++ <attribute name="type"> <text />
</attribute>
++ </element>
++ </optional>
++ <optional>
++ <element name="current_dc">
++ <attribute name="present"> <data
type="boolean" /> </attribute>
++ <optional>
++ <group>
++ <attribute name="version"> <text />
</attribute>
++ <attribute name="name"> <text />
</attribute>
++ <attribute name="id"> <text />
</attribute>
++ <attribute name="with_quorum"> <data
type="boolean" /> </attribute>
++ </group>
++ </optional>
++ </element>
++ </optional>
++ <optional>
++ <element name="last_update">
++ <attribute name="time"> <text />
</attribute>
++ </element>
++ <element name="last_change">
++ <attribute name="time"> <text />
</attribute>
++ <attribute name="user"> <text />
</attribute>
++ <attribute name="client"> <text />
</attribute>
++ <attribute name="origin"> <text />
</attribute>
++ </element>
++ </optional>
++ <optional>
++ <element name="nodes_configured">
++ <attribute name="number"> <data
type="nonNegativeInteger" /> </attribute>
++ </element>
++ <element name="resources_configured">
++ <attribute name="number"> <data
type="nonNegativeInteger" /> </attribute>
++ <attribute name="disabled"> <data
type="nonNegativeInteger" /> </attribute>
++ <attribute name="blocked"> <data
type="nonNegativeInteger" /> </attribute>
++ </element>
++ </optional>
++ <optional>
++ <element name="cluster_options">
++ <attribute name="stonith-enabled"> <data
type="boolean" /> </attribute>
++ <attribute name="symmetric-cluster"> <data
type="boolean" /> </attribute>
++ <attribute name="no-quorum-policy"> <text />
</attribute>
++ <attribute name="maintenance-mode"> <data
type="boolean" /> </attribute>
++ <attribute name="stop-all-resources"> <data
type="boolean" /> </attribute>
++ </element>
++ </optional>
++ </element>
++ </define>
++
++ <define name="resources-list">
++ <element name="resources">
++ <zeroOrMore>
++ <externalRef href="resources-2.4.rng" />
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="nodes-list">
++ <element name="nodes">
++ <zeroOrMore>
++ <externalRef href="nodes-2.8.rng" />
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="node-attributes-list">
++ <element name="node_attributes">
++ <zeroOrMore>
++ <externalRef href="node-attrs-2.8.rng" />
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="node-history-list">
++ <element name="node_history">
++ <zeroOrMore>
++ <ref name="element-node-history" />
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="failures-list">
++ <element name="failures">
++ <zeroOrMore>
++ <externalRef href="failure-2.8.rng" />
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="fence-event-list">
++ <element name="fence_history">
++ <optional>
++ <attribute name="status"> <data
type="integer" /> </attribute>
++ </optional>
++ <zeroOrMore>
++ <externalRef href="fence-event-2.0.rng" />
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="tickets-list">
++ <element name="tickets">
++ <zeroOrMore>
++ <ref name="element-ticket" />
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="bans-list">
++ <element name="bans">
++ <zeroOrMore>
++ <ref name="element-ban" />
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="element-node-history">
++ <element name="node">
++ <attribute name="name"> <text /> </attribute>
++ <zeroOrMore>
++ <ref name="element-resource-history" />
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="element-resource-history">
++ <element name="resource_history">
++ <attribute name="id"> <text /> </attribute>
++ <attribute name="orphan"> <data type="boolean"
/> </attribute>
++ <optional>
++ <group>
++ <attribute name="migration-threshold"> <data
type="nonNegativeInteger" /> </attribute>
++ <optional>
++ <attribute name="fail-count"> <text />
</attribute>
++ </optional>
++ <optional>
++ <attribute name="last-failure"> <text />
</attribute>
++ </optional>
++ </group>
++ </optional>
++ <zeroOrMore>
++ <ref name="element-operation-history" />
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="element-operation-history">
++ <element name="operation_history">
++ <attribute name="call"> <text /> </attribute>
++ <attribute name="task"> <text /> </attribute>
++ <optional>
++ <attribute name="interval"> <text />
</attribute>
++ </optional>
++ <optional>
++ <attribute name="last-rc-change"> <text />
</attribute>
++ </optional>
++ <optional>
++ <attribute name="last-run"> <text />
</attribute>
++ </optional>
++ <optional>
++ <attribute name="exec-time"> <text />
</attribute>
++ </optional>
++ <optional>
++ <attribute name="queue-time"> <text />
</attribute>
++ </optional>
++ <attribute name="rc"> <data type="integer"
/> </attribute>
++ <attribute name="rc_text"> <text />
</attribute>
++ </element>
++ </define>
++
++ <define name="element-ticket">
++ <element name="ticket">
++ <attribute name="id"> <text /> </attribute>
++ <attribute name="status">
++ <choice>
++ <value>granted</value>
++ <value>revoked</value>
++ </choice>
++ </attribute>
++ <attribute name="standby"> <data type="boolean"
/> </attribute>
++ <optional>
++ <attribute name="last-granted"> <text />
</attribute>
++ </optional>
++ </element>
++ </define>
++
++ <define name="element-ban">
++ <element name="ban">
++ <attribute name="id"> <text /> </attribute>
++ <attribute name="resource"> <text />
</attribute>
++ <attribute name="node"> <text /> </attribute>
++ <attribute name="weight"> <data type="integer"
/> </attribute>
++ <attribute name="promoted-only"> <data
type="boolean" /> </attribute>
++ <!-- DEPRECATED: master_only is a duplicate of promoted-only that is
++ provided solely for API backward compatibility. It will be
++ removed in a future release. Check promoted-only instead.
++ -->
++ <attribute name="master_only"> <data
type="boolean" /> </attribute>
++ </element>
++ </define>
++</grammar>
+--
+1.8.3.1
+
+
+From da394983f106f974274ddd94675a04c85086010e Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Fri, 18 Jun 2021 15:06:34 -0400
+Subject: [PATCH 2/5] Refactor: Split node history out into its own XML schema.
+
+This allows for sharing it between crm_mon and crm_simulate.
+---
+ xml/Makefile.am | 2 +-
+ xml/api/crm_mon-2.12.rng | 64 +--------------------------------------
+ xml/api/node-history-2.12.rng | 70 +++++++++++++++++++++++++++++++++++++++++++
+ 3 files changed, 72 insertions(+), 64 deletions(-)
+ create mode 100644 xml/api/node-history-2.12.rng
+
+diff --git a/xml/Makefile.am b/xml/Makefile.am
+index b9448d4..8e7b6d3 100644
+--- a/xml/Makefile.am
++++ b/xml/Makefile.am
+@@ -64,7 +64,7 @@ API_request_base = command-output \
+ CIB_cfg_base = options nodes resources constraints fencing acls tags alerts
+
+ # Names of all schemas (including top level and those included by others)
+-API_base = $(API_request_base) fence-event failure generic-list item node-attrs nodes
resources status
++API_base = $(API_request_base) fence-event failure generic-list item node-attrs
node-history nodes resources status
+ CIB_base = cib $(CIB_cfg_base) status score rule nvset
+
+ # Static schema files and transforms (only CIB has transforms)
+diff --git a/xml/api/crm_mon-2.12.rng b/xml/api/crm_mon-2.12.rng
+index ffec923..be14412 100644
+--- a/xml/api/crm_mon-2.12.rng
++++ b/xml/api/crm_mon-2.12.rng
+@@ -20,7 +20,7 @@
+ <ref name="node-attributes-list" />
+ </optional>
+ <optional>
+- <ref name="node-history-list" />
++ <externalRef href="node-history-2.12.rng"/>
+ </optional>
+ <optional>
+ <ref name="failures-list" />
+@@ -113,14 +113,6 @@
+ </element>
+ </define>
+
+- <define name="node-history-list">
+- <element name="node_history">
+- <zeroOrMore>
+- <ref name="element-node-history" />
+- </zeroOrMore>
+- </element>
+- </define>
+-
+ <define name="failures-list">
+ <element name="failures">
+ <zeroOrMore>
+@@ -156,60 +148,6 @@
+ </element>
+ </define>
+
+- <define name="element-node-history">
+- <element name="node">
+- <attribute name="name"> <text /> </attribute>
+- <zeroOrMore>
+- <ref name="element-resource-history" />
+- </zeroOrMore>
+- </element>
+- </define>
+-
+- <define name="element-resource-history">
+- <element name="resource_history">
+- <attribute name="id"> <text /> </attribute>
+- <attribute name="orphan"> <data type="boolean"
/> </attribute>
+- <optional>
+- <group>
+- <attribute name="migration-threshold"> <data
type="nonNegativeInteger" /> </attribute>
+- <optional>
+- <attribute name="fail-count"> <text />
</attribute>
+- </optional>
+- <optional>
+- <attribute name="last-failure"> <text />
</attribute>
+- </optional>
+- </group>
+- </optional>
+- <zeroOrMore>
+- <ref name="element-operation-history" />
+- </zeroOrMore>
+- </element>
+- </define>
+-
+- <define name="element-operation-history">
+- <element name="operation_history">
+- <attribute name="call"> <text /> </attribute>
+- <attribute name="task"> <text /> </attribute>
+- <optional>
+- <attribute name="interval"> <text />
</attribute>
+- </optional>
+- <optional>
+- <attribute name="last-rc-change"> <text />
</attribute>
+- </optional>
+- <optional>
+- <attribute name="last-run"> <text />
</attribute>
+- </optional>
+- <optional>
+- <attribute name="exec-time"> <text />
</attribute>
+- </optional>
+- <optional>
+- <attribute name="queue-time"> <text />
</attribute>
+- </optional>
+- <attribute name="rc"> <data type="integer"
/> </attribute>
+- <attribute name="rc_text"> <text />
</attribute>
+- </element>
+- </define>
+-
+ <define name="element-ticket">
+ <element name="ticket">
+ <attribute name="id"> <text /> </attribute>
+diff --git a/xml/api/node-history-2.12.rng b/xml/api/node-history-2.12.rng
+new file mode 100644
+index 0000000..9628000
+--- /dev/null
++++ b/xml/api/node-history-2.12.rng
+@@ -0,0 +1,70 @@
++<?xml version="1.0" encoding="UTF-8"?>
++<grammar
xmlns="http://relaxng.org/ns/structure/1.0"
++
datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
++
++ <start>
++ <ref name="node-history-list" />
++ </start>
++
++ <define name="node-history-list">
++ <element name="node_history">
++ <zeroOrMore>
++ <ref name="element-node-history" />
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="element-node-history">
++ <element name="node">
++ <attribute name="name"> <text /> </attribute>
++ <zeroOrMore>
++ <ref name="element-resource-history" />
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="element-resource-history">
++ <element name="resource_history">
++ <attribute name="id"> <text /> </attribute>
++ <attribute name="orphan"> <data type="boolean"
/> </attribute>
++ <optional>
++ <group>
++ <attribute name="migration-threshold"> <data
type="nonNegativeInteger" /> </attribute>
++ <optional>
++ <attribute name="fail-count"> <text />
</attribute>
++ </optional>
++ <optional>
++ <attribute name="last-failure"> <text />
</attribute>
++ </optional>
++ </group>
++ </optional>
++ <zeroOrMore>
++ <ref name="element-operation-history" />
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="element-operation-history">
++ <element name="operation_history">
++ <attribute name="call"> <text /> </attribute>
++ <attribute name="task"> <text /> </attribute>
++ <optional>
++ <attribute name="interval"> <text />
</attribute>
++ </optional>
++ <optional>
++ <attribute name="last-rc-change"> <text />
</attribute>
++ </optional>
++ <optional>
++ <attribute name="last-run"> <text />
</attribute>
++ </optional>
++ <optional>
++ <attribute name="exec-time"> <text />
</attribute>
++ </optional>
++ <optional>
++ <attribute name="queue-time"> <text />
</attribute>
++ </optional>
++ <attribute name="rc"> <data type="integer"
/> </attribute>
++ <attribute name="rc_text"> <text />
</attribute>
++ </element>
++ </define>
++</grammar>
+--
+1.8.3.1
+
+
+From bf72b2615630eef7876e443d60b34d5a316de847 Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Wed, 16 Jun 2021 14:09:31 -0400
+Subject: [PATCH 3/5] Low: schemas: Copy crm_simulate.rng in preparation for
+ changes.
+
+---
+ xml/api/crm_simulate-2.12.rng | 335 ++++++++++++++++++++++++++++++++++++++++++
+ 1 file changed, 335 insertions(+)
+ create mode 100644 xml/api/crm_simulate-2.12.rng
+
+diff --git a/xml/api/crm_simulate-2.12.rng b/xml/api/crm_simulate-2.12.rng
+new file mode 100644
+index 0000000..9a7612d
+--- /dev/null
++++ b/xml/api/crm_simulate-2.12.rng
+@@ -0,0 +1,335 @@
++<?xml version="1.0" encoding="UTF-8"?>
++<grammar
xmlns="http://relaxng.org/ns/structure/1.0"
++
datatypeLibrary="http://www.w3.org/2001/XMLSchema-datatypes">
++
++ <start>
++ <ref name="element-crm-simulate"/>
++ </start>
++
++ <define name="element-crm-simulate">
++ <choice>
++ <ref name="timings-list" />
++ <group>
++ <ref name="cluster-status" />
++ <optional>
++ <ref name="modifications-list" />
++ </optional>
++ <optional>
++ <ref name="allocations-utilizations-list" />
++ </optional>
++ <optional>
++ <ref name="action-list" />
++ </optional>
++ <optional>
++ <ref name="cluster-injected-actions-list" />
++ <ref name="revised-cluster-status" />
++ </optional>
++ </group>
++ </choice>
++ </define>
++
++ <define name="allocations-utilizations-list">
++ <choice>
++ <element name="allocations">
++ <zeroOrMore>
++ <choice>
++ <ref name="element-allocation" />
++ <ref name="element-promotion" />
++ </choice>
++ </zeroOrMore>
++ </element>
++ <element name="utilizations">
++ <zeroOrMore>
++ <choice>
++ <ref name="element-capacity" />
++ <ref name="element-utilization" />
++ </choice>
++ </zeroOrMore>
++ </element>
++ <element name="allocations_utilizations">
++ <zeroOrMore>
++ <choice>
++ <ref name="element-allocation" />
++ <ref name="element-promotion" />
++ <ref name="element-capacity" />
++ <ref name="element-utilization" />
++ </choice>
++ </zeroOrMore>
++ </element>
++ </choice>
++ </define>
++
++ <define name="cluster-status">
++ <element name="cluster_status">
++ <ref name="nodes-list" />
++ <ref name="resources-list" />
++ <optional>
++ <ref name="node-attributes-list" />
++ </optional>
++ <optional>
++ <ref name="failures-list" />
++ </optional>
++ </element>
++ </define>
++
++ <define name="modifications-list">
++ <element name="modifications">
++ <optional>
++ <attribute name="quorum"> <text />
</attribute>
++ </optional>
++ <optional>
++ <attribute name="watchdog"> <text />
</attribute>
++ </optional>
++ <zeroOrMore>
++ <ref name="element-inject-modify-node" />
++ </zeroOrMore>
++ <zeroOrMore>
++ <ref name="element-inject-modify-ticket" />
++ </zeroOrMore>
++ <zeroOrMore>
++ <ref name="element-inject-spec" />
++ </zeroOrMore>
++ <zeroOrMore>
++ <ref name="element-inject-attr" />
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="revised-cluster-status">
++ <element name="revised_cluster_status">
++ <ref name="nodes-list" />
++ <ref name="resources-list" />
++ <optional>
++ <ref name="node-attributes-list" />
++ </optional>
++ <optional>
++ <ref name="failures-list" />
++ </optional>
++ </element>
++ </define>
++
++ <define name="element-inject-attr">
++ <element name="inject_attr">
++ <attribute name="cib_node"> <text />
</attribute>
++ <attribute name="name"> <text /> </attribute>
++ <attribute name="node_path"> <text />
</attribute>
++ <attribute name="value"> <text /> </attribute>
++ </element>
++ </define>
++
++ <define name="element-inject-modify-node">
++ <element name="modify_node">
++ <attribute name="action"> <text /> </attribute>
++ <attribute name="node"> <text /> </attribute>
++ </element>
++ </define>
++
++ <define name="element-inject-spec">
++ <element name="inject_spec">
++ <attribute name="spec"> <text /> </attribute>
++ </element>
++ </define>
++
++ <define name="element-inject-modify-ticket">
++ <element name="modify_ticket">
++ <attribute name="action"> <text /> </attribute>
++ <attribute name="ticket"> <text /> </attribute>
++ </element>
++ </define>
++
++ <define name="cluster-injected-actions-list">
++ <element name="transition">
++ <zeroOrMore>
++ <ref name="element-injected-actions" />
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="node-attributes-list">
++ <element name="node_attributes">
++ <zeroOrMore>
++ <externalRef href="node-attrs-2.8.rng" />
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="failures-list">
++ <element name="failures">
++ <zeroOrMore>
++ <externalRef href="failure-2.8.rng" />
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="nodes-list">
++ <element name="nodes">
++ <zeroOrMore>
++ <externalRef href="nodes-2.8.rng" />
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="resources-list">
++ <element name="resources">
++ <zeroOrMore>
++ <externalRef href="resources-2.4.rng" />
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="timings-list">
++ <element name="timings">
++ <zeroOrMore>
++ <ref name="element-timing" />
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="action-list">
++ <element name="actions">
++ <zeroOrMore>
++ <ref name="element-node-action" />
++ </zeroOrMore>
++ <zeroOrMore>
++ <ref name="element-rsc-action" />
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="element-allocation">
++ <element name="node_weight">
++ <attribute name="function"> <text />
</attribute>
++ <attribute name="node"> <text /> </attribute>
++ <externalRef href="../score.rng" />
++ <optional>
++ <attribute name="id"> <text /> </attribute>
++ </optional>
++ </element>
++ </define>
++
++ <define name="element-capacity">
++ <element name="capacity">
++ <attribute name="comment"> <text />
</attribute>
++ <attribute name="node"> <text /> </attribute>
++ <zeroOrMore>
++ <element>
++ <anyName />
++ <text />
++ </element>
++ </zeroOrMore>
++ </element>
++ </define>
++
++ <define name="element-inject-cluster-action">
++ <element name="cluster_action">
++ <attribute name="node"> <text /> </attribute>
++ <attribute name="task"> <text /> </attribute>
++ <optional>
++ <attribute name="id"> <text /> </attribute>
++ </optional>
++ </element>
++ </define>
++
++ <define name="element-injected-actions">
++ <choice>
++ <ref name="element-inject-cluster-action" />
++ <ref name="element-inject-fencing-action" />
++ <ref name="element-inject-pseudo-action" />
++ <ref name="element-inject-rsc-action" />
++ </choice>
++ </define>
++
++ <define name="element-inject-fencing-action">
++ <element name="fencing_action">
++ <attribute name="op"> <text /> </attribute>
++ <attribute name="target"> <text /> </attribute>
++ </element>
++ </define>
++
++ <define name="element-node-action">
++ <element name="node_action">
++ <attribute name="node"> <text /> </attribute>
++ <attribute name="reason"> <text /> </attribute>
++ <attribute name="task"> <text /> </attribute>
++ </element>
++ </define>
++
++ <define name="element-promotion">
++ <element name="promotion_score">
++ <attribute name="id"> <text /> </attribute>
++ <externalRef href="../score.rng" />
++ <optional>
++ <attribute name="node"> <text />
</attribute>
++ </optional>
++ </element>
++ </define>
++
++ <define name="element-inject-pseudo-action">
++ <element name="pseudo_action">
++ <attribute name="task"> <text /> </attribute>
++ <optional>
++ <attribute name="node"> <text />
</attribute>
++ </optional>
++ </element>
++ </define>
++
++ <define name="element-inject-rsc-action">
++ <element name="rsc_action">
++ <attribute name="node"> <text /> </attribute>
++ <attribute name="op"> <text /> </attribute>
++ <attribute name="resource"> <text />
</attribute>
++ <optional>
++ <attribute name="interval"> <data
type="integer" /> </attribute>
++ </optional>
++ </element>
++ </define>
++
++ <define name="element-timing">
++ <element name="timing">
++ <attribute name="file"> <text /> </attribute>
++ <attribute name="duration"> <data type="double"
/> </attribute>
++ </element>
++ </define>
++
++ <define name="element-rsc-action">
++ <element name="rsc_action">
++ <attribute name="action"> <text /> </attribute>
++ <attribute name="resource"> <text />
</attribute>
++ <optional>
++ <attribute name="blocked"> <data
type="boolean" /> </attribute>
++ </optional>
++ <optional>
++ <attribute name="dest"> <text />
</attribute>
++ </optional>
++ <optional>
++ <attribute name="next-role"> <text />
</attribute>
++ </optional>
++ <optional>
++ <attribute name="node"> <text />
</attribute>
++ </optional>
++ <optional>
++ <attribute name="reason"> <text />
</attribute>
++ </optional>
++ <optional>
++ <attribute name="role"> <text />
</attribute>
++ </optional>
++ <optional>
++ <attribute name="source"> <text />
</attribute>
++ </optional>
++ </element>
++ </define>
++
++ <define name="element-utilization">
++ <element name="utilization">
++ <attribute name="function"> <text />
</attribute>
++ <attribute name="node"> <text /> </attribute>
++ <attribute name="resource"> <text />
</attribute>
++ <zeroOrMore>
++ <element>
++ <anyName />
++ <text />
++ </element>
++ </zeroOrMore>
++ </element>
++ </define>
++</grammar>
+--
+1.8.3.1
+
+
+From c46e07788788acf5669e3f89b9344190a91c7331 Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Fri, 18 Jun 2021 15:10:19 -0400
+Subject: [PATCH 4/5] Feature: tools: Add the node-summary to crm_simulate
+ output.
+
+If --show-failcounts is given to crm_simulate, it should also display
+the node-summary message.
+
+See: rhbz#1686426
+---
+ tools/crm_simulate.c | 7 +++++--
+ xml/api/crm_simulate-2.12.rng | 3 +++
+ 2 files changed, 8 insertions(+), 2 deletions(-)
+
+diff --git a/tools/crm_simulate.c b/tools/crm_simulate.c
+index b4aa9d1..2ea292c 100644
+--- a/tools/crm_simulate.c
++++ b/tools/crm_simulate.c
+@@ -409,11 +409,14 @@ print_cluster_status(pe_working_set_t * data_set, unsigned int
print_opts)
+ FALSE, FALSE, all, all, FALSE);
+
+ if (options.show_attrs) {
+- out->message(out, "node-attribute-list", data_set,
+- 0, rc == pcmk_rc_ok, FALSE, FALSE, FALSE, all, all);
++ rc = out->message(out, "node-attribute-list", data_set,
++ 0, rc == pcmk_rc_ok, FALSE, FALSE, FALSE, all, all);
+ }
+
+ if (options.show_failcounts) {
++ rc = out->message(out, "node-summary", data_set, all, all,
++ 0, print_opts, FALSE, FALSE, FALSE, FALSE, rc == pcmk_rc_ok);
++
+ out->message(out, "failed-action-list", data_set, all, all,
+ rc == pcmk_rc_ok);
+ }
+diff --git a/xml/api/crm_simulate-2.12.rng b/xml/api/crm_simulate-2.12.rng
+index 9a7612d..f90bd36 100644
+--- a/xml/api/crm_simulate-2.12.rng
++++ b/xml/api/crm_simulate-2.12.rng
+@@ -67,6 +67,9 @@
+ <ref name="node-attributes-list" />
+ </optional>
+ <optional>
++ <externalRef href="node-history-2.12.rng" />
++ </optional>
++ <optional>
+ <ref name="failures-list" />
+ </optional>
+ </element>
+--
+1.8.3.1
+
+
+From bac50336e0264604716e5997b87ee7e65311b982 Mon Sep 17 00:00:00 2001
+From: Chris Lumens <clumens(a)redhat.com>
+Date: Fri, 18 Jun 2021 15:21:52 -0400
+Subject: [PATCH 5/5] Low: libcrmcommon: Increase PCMK__API_VERSION for new
+ crm_resource output.
+
+See: rhbz#1686426
+---
+ include/crm/common/output_internal.h | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/include/crm/common/output_internal.h b/include/crm/common/output_internal.h
+index 0436cde..ba9c423 100644
+--- a/include/crm/common/output_internal.h
++++ b/include/crm/common/output_internal.h
+@@ -27,7 +27,7 @@ extern "C" {
+ # include <glib.h>
+ # include <crm/common/results.h>
+
+-# define PCMK__API_VERSION "2.11"
++# define PCMK__API_VERSION "2.12"
+
+ #if defined(PCMK__WITH_ATTRIBUTE_OUTPUT_ARGS)
+ # define PCMK__OUTPUT_ARGS(ARGS...) __attribute__((output_args(ARGS)))
+--
+1.8.3.1
+
diff --git a/007-unfencing-loop.patch b/007-unfencing-loop.patch
new file mode 100644
index 0000000..d4950c8
--- /dev/null
+++ b/007-unfencing-loop.patch
@@ -0,0 +1,733 @@
+From 6dcd6b51d7d3993bc483588d6ed75077518ed600 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot(a)redhat.com>
+Date: Fri, 4 Jun 2021 16:30:55 -0500
+Subject: [PATCH 01/11] Low: controller: check whether unfenced node was remote
+ node
+
+... so the controller can indicate the node is remote (if known at that point,
+which is not guaranteed) when setting unfencing-related node attributes.
+---
+ daemons/controld/controld_fencing.c | 21 ++++++++++++++++++---
+ 1 file changed, 18 insertions(+), 3 deletions(-)
+
+diff --git a/daemons/controld/controld_fencing.c b/daemons/controld/controld_fencing.c
+index 23dff28..0fba661 100644
+--- a/daemons/controld/controld_fencing.c
++++ b/daemons/controld/controld_fencing.c
+@@ -757,15 +757,30 @@ tengine_stonith_callback(stonith_t *stonith,
stonith_callback_data_t *data)
+ if (pcmk__str_eq("on", op, pcmk__str_casei)) {
+ const char *value = NULL;
+ char *now = pcmk__ttoa(time(NULL));
++ gboolean is_remote_node = FALSE;
++
++ /* This check is not 100% reliable, since this node is not
++ * guaranteed to have the remote node cached. However, it
++ * doesn't have to be reliable, since the attribute manager can
++ * learn a node's "remoteness" by other means sooner or
later.
++ * This allows it to learn more quickly if this node does have
++ * the information.
++ */
++ if (g_hash_table_lookup(crm_remote_peer_cache, uuid) != NULL) {
++ is_remote_node = TRUE;
++ }
+
+- update_attrd(target, CRM_ATTR_UNFENCED, now, NULL, FALSE);
++ update_attrd(target, CRM_ATTR_UNFENCED, now, NULL,
++ is_remote_node);
+ free(now);
+
+ value = crm_meta_value(action->params, XML_OP_ATTR_DIGESTS_ALL);
+- update_attrd(target, CRM_ATTR_DIGESTS_ALL, value, NULL, FALSE);
++ update_attrd(target, CRM_ATTR_DIGESTS_ALL, value, NULL,
++ is_remote_node);
+
+ value = crm_meta_value(action->params, XML_OP_ATTR_DIGESTS_SECURE);
+- update_attrd(target, CRM_ATTR_DIGESTS_SECURE, value, NULL, FALSE);
++ update_attrd(target, CRM_ATTR_DIGESTS_SECURE, value, NULL,
++ is_remote_node);
+
+ } else if (action->sent_update == FALSE) {
+ send_stonith_update(action, target, uuid);
+--
+1.8.3.1
+
+
+From 3ef6d9403f68ab8559c45cc99f5a8da05ca6420b Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot(a)redhat.com>
+Date: Mon, 7 Jun 2021 10:50:36 -0500
+Subject: [PATCH 02/11] Refactor: pacemaker-attrd: functionize adding remote
+ node to cache
+
+... for future reuse
+---
+ daemons/attrd/attrd_commands.c | 34 +++++++++++++++++++++++-----------
+ 1 file changed, 23 insertions(+), 11 deletions(-)
+
+diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c
+index 731c243..93a165b 100644
+--- a/daemons/attrd/attrd_commands.c
++++ b/daemons/attrd/attrd_commands.c
+@@ -102,6 +102,28 @@ free_attribute(gpointer data)
+ }
+ }
+
++/*!
++ * \internal
++ * \brief Ensure a Pacemaker Remote node is in the correct peer cache
++ *
++ * \param[in]
++ */
++static void
++cache_remote_node(const char *node_name)
++{
++ /* If we previously assumed this node was an unseen cluster node,
++ * remove its entry from the cluster peer cache.
++ */
++ crm_node_t *dup = pcmk__search_cluster_node_cache(0, node_name);
++
++ if (dup && (dup->uuid == NULL)) {
++ reap_crm_member(0, node_name);
++ }
++
++ // Ensure node is in the remote peer cache
++ CRM_ASSERT(crm_remote_peer_get(node_name) != NULL);
++}
++
+ static xmlNode *
+ build_attribute_xml(
+ xmlNode *parent, const char *name, const char *set, const char *uuid, unsigned int
timeout_ms, const char *user,
+@@ -709,17 +731,7 @@ attrd_lookup_or_create_value(GHashTable *values, const char *host,
xmlNode *xml)
+
+ crm_element_value_int(xml, PCMK__XA_ATTR_IS_REMOTE, &is_remote);
+ if (is_remote) {
+- /* If we previously assumed this node was an unseen cluster node,
+- * remove its entry from the cluster peer cache.
+- */
+- crm_node_t *dup = pcmk__search_cluster_node_cache(0, host);
+-
+- if (dup && (dup->uuid == NULL)) {
+- reap_crm_member(0, host);
+- }
+-
+- /* Ensure this host is in the remote peer cache */
+- CRM_ASSERT(crm_remote_peer_get(host) != NULL);
++ cache_remote_node(host);
+ }
+
+ if (v == NULL) {
+--
+1.8.3.1
+
+
+From 6fac2c71bc2c56870ac828d7cd7b7c799279c47e Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot(a)redhat.com>
+Date: Mon, 7 Jun 2021 10:39:34 -0500
+Subject: [PATCH 03/11] Refactor: pacemaker-attrd: don't try to remove votes
+ for remote nodes
+
+Remote nodes never vote.
+
+This has no effect in practice since the removal would simply do nothing,
+but we might as well not waste time trying.
+---
+ daemons/attrd/attrd_commands.c | 11 ++++++-----
+ 1 file changed, 6 insertions(+), 5 deletions(-)
+
+diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c
+index 93a165b..dbe777e 100644
+--- a/daemons/attrd/attrd_commands.c
++++ b/daemons/attrd/attrd_commands.c
+@@ -976,7 +976,8 @@ attrd_election_cb(gpointer user_data)
+ void
+ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *data)
+ {
+- bool remove_voter = FALSE;
++ bool gone = false;
++ bool is_remote = pcmk_is_set(peer->flags, crm_remote_node);
+
+ switch (kind) {
+ case crm_status_uname:
+@@ -984,7 +985,7 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer,
const void *da
+
+ case crm_status_processes:
+ if (!pcmk_is_set(peer->processes, crm_get_cluster_proc())) {
+- remove_voter = TRUE;
++ gone = true;
+ }
+ break;
+
+@@ -1000,13 +1001,13 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer,
const void *da
+ } else {
+ // Remove all attribute values associated with lost nodes
+ attrd_peer_remove(peer->uname, FALSE, "loss");
+- remove_voter = TRUE;
++ gone = true;
+ }
+ break;
+ }
+
+- // In case an election is in progress, remove any vote by the node
+- if (remove_voter) {
++ // Remove votes from cluster nodes that leave, in case election in progress
++ if (gone && !is_remote) {
+ attrd_remove_voter(peer);
+ }
+ }
+--
+1.8.3.1
+
+
+From 54089fc663d6aaf10ca164c6c94b3b17237788de Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot(a)redhat.com>
+Date: Mon, 7 Jun 2021 10:40:06 -0500
+Subject: [PATCH 04/11] Low: pacemaker-attrd: check for remote nodes in peer
+ update callback
+
+If a remote node was started before the local cluster node joined the cluster,
+the cluster node will assume its node attributes are for a cluster node until
+it learns otherwise. Check for remoteness in the peer update callback, to have
+another way we can learn it.
+---
+ daemons/attrd/attrd_commands.c | 4 ++++
+ 1 file changed, 4 insertions(+)
+
+diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c
+index dbe777e..5f6a754 100644
+--- a/daemons/attrd/attrd_commands.c
++++ b/daemons/attrd/attrd_commands.c
+@@ -1009,6 +1009,10 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer,
const void *da
+ // Remove votes from cluster nodes that leave, in case election in progress
+ if (gone && !is_remote) {
+ attrd_remove_voter(peer);
++
++ // Ensure remote nodes that come up are in the remote node cache
++ } else if (!gone && is_remote) {
++ cache_remote_node(peer->uname);
+ }
+ }
+
+--
+1.8.3.1
+
+
+From 8c048df0312d0d9c857d87b570a352429a710928 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot(a)redhat.com>
+Date: Mon, 7 Jun 2021 11:29:12 -0500
+Subject: [PATCH 05/11] Log: pacemaker-attrd: log peer status changes
+
+---
+ daemons/attrd/attrd_commands.c | 9 +++++++++
+ 1 file changed, 9 insertions(+)
+
+diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c
+index 5f6a754..d6d179b 100644
+--- a/daemons/attrd/attrd_commands.c
++++ b/daemons/attrd/attrd_commands.c
+@@ -972,6 +972,7 @@ attrd_election_cb(gpointer user_data)
+ return FALSE;
+ }
+
++#define state_text(state) ((state)? (const char *)(state) : "in unknown
state")
+
+ void
+ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer, const void *data)
+@@ -981,15 +982,23 @@ attrd_peer_change_cb(enum crm_status_type kind, crm_node_t *peer,
const void *da
+
+ switch (kind) {
+ case crm_status_uname:
++ crm_debug("%s node %s is now %s",
++ (is_remote? "Remote" : "Cluster"),
++ peer->uname, state_text(peer->state));
+ break;
+
+ case crm_status_processes:
+ if (!pcmk_is_set(peer->processes, crm_get_cluster_proc())) {
+ gone = true;
+ }
++ crm_debug("Node %s is %s a peer",
++ peer->uname, (gone? "no longer" : "now"));
+ break;
+
+ case crm_status_nstate:
++ crm_debug("%s node %s is now %s (was %s)",
++ (is_remote? "Remote" : "Cluster"),
++ peer->uname, state_text(peer->state), state_text(data));
+ if (pcmk__str_eq(peer->state, CRM_NODE_MEMBER, pcmk__str_casei)) {
+ /* If we're the writer, send new peers a list of all attributes
+ * (unless it's a remote node, which doesn't run its own attrd)
+--
+1.8.3.1
+
+
+From 1dcc8dee4990cf0dbdec0e14db6d9a3ad67a41d5 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot(a)redhat.com>
+Date: Mon, 7 Jun 2021 11:13:53 -0500
+Subject: [PATCH 06/11] Low: pacemaker-attrd: ensure node ID is only set for
+ attributes when known
+
+In most cases, attribute updates contained the node ID, and the node ID was
+used by other code, only if known (i.e. positive). However a couple places did
+not check this, so add that.
+
+I am unsure whether the missing check caused problems in practice, but there
+appears to be the possibility that a remote node would wrongly be added to the
+cluster node cache.
+---
+ daemons/attrd/attrd_commands.c | 6 ++++--
+ 1 file changed, 4 insertions(+), 2 deletions(-)
+
+diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c
+index d6d179b..b3f441c 100644
+--- a/daemons/attrd/attrd_commands.c
++++ b/daemons/attrd/attrd_commands.c
+@@ -136,7 +136,9 @@ build_attribute_xml(
+ crm_xml_add(xml, PCMK__XA_ATTR_UUID, uuid);
+ crm_xml_add(xml, PCMK__XA_ATTR_USER, user);
+ crm_xml_add(xml, PCMK__XA_ATTR_NODE_NAME, peer);
+- crm_xml_add_int(xml, PCMK__XA_ATTR_NODE_ID, peerid);
++ if (peerid > 0) {
++ crm_xml_add_int(xml, PCMK__XA_ATTR_NODE_ID, peerid);
++ }
+ crm_xml_add(xml, PCMK__XA_ATTR_VALUE, value);
+ crm_xml_add_int(xml, PCMK__XA_ATTR_DAMPENING, timeout_ms/1000);
+ crm_xml_add_int(xml, PCMK__XA_ATTR_IS_PRIVATE, is_private);
+@@ -937,7 +939,7 @@ attrd_peer_update(crm_node_t *peer, xmlNode *xml, const char *host,
bool filter)
+ /* If this is a cluster node whose node ID we are learning, remember it */
+ if ((v->nodeid == 0) && (v->is_remote == FALSE)
+ && (crm_element_value_int(xml, PCMK__XA_ATTR_NODE_ID,
+- (int*)&v->nodeid) == 0)) {
++ (int*)&v->nodeid) == 0) &&
(v->nodeid > 0)) {
+
+ crm_node_t *known_peer = crm_get_peer(v->nodeid, host);
+
+--
+1.8.3.1
+
+
+From 8d12490e88b558d01db37a38f7d35175c6d2d69a Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot(a)redhat.com>
+Date: Thu, 10 Jun 2021 17:25:57 -0500
+Subject: [PATCH 07/11] Refactor: pacemaker-attrd: functionize processing a
+ sync response
+
+... for code isolation, and because we need to add more to it
+---
+ daemons/attrd/attrd_commands.c | 59 ++++++++++++++++++++++++++++--------------
+ 1 file changed, 39 insertions(+), 20 deletions(-)
+
+diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c
+index b3f441c..d02d3e6 100644
+--- a/daemons/attrd/attrd_commands.c
++++ b/daemons/attrd/attrd_commands.c
+@@ -572,6 +572,43 @@ attrd_peer_clear_failure(crm_node_t *peer, xmlNode *xml)
+ }
+
+ /*!
++ * \internal
++ * \brief Load attributes from a peer sync response
++ *
++ * \param[in] peer Peer that sent clear request
++ * \param[in] peer_won Whether peer is the attribute writer
++ * \param[in] xml Request XML
++ */
++static void
++process_peer_sync_response(crm_node_t *peer, bool peer_won, xmlNode *xml)
++{
++ crm_info("Processing " PCMK__ATTRD_CMD_SYNC_RESPONSE " from
%s",
++ peer->uname);
++
++ if (peer_won) {
++ /* Initialize the "seen" flag for all attributes to cleared, so we
can
++ * detect attributes that local node has but the writer doesn't.
++ */
++ clear_attribute_value_seen();
++ }
++
++ // Process each attribute update in the sync response
++ for (xmlNode *child = pcmk__xml_first_child(xml); child != NULL;
++ child = pcmk__xml_next(child)) {
++ attrd_peer_update(peer, child,
++ crm_element_value(child, PCMK__XA_ATTR_NODE_NAME),
++ TRUE);
++ }
++
++ if (peer_won) {
++ /* If any attributes are still not marked as seen, the writer doesn't
++ * know about them, so send all peers an update with them.
++ */
++ attrd_current_only_attribute_update(peer, xml);
++ }
++}
++
++/*!
+ \internal
+ \brief Broadcast private attribute for local node with protocol version
+ */
+@@ -596,7 +633,7 @@ attrd_peer_message(crm_node_t *peer, xmlNode *xml)
+ const char *op = crm_element_value(xml, PCMK__XA_TASK);
+ const char *election_op = crm_element_value(xml, F_CRM_TASK);
+ const char *host = crm_element_value(xml, PCMK__XA_ATTR_NODE_NAME);
+- bool peer_won = FALSE;
++ bool peer_won = false;
+
+ if (election_op) {
+ attrd_handle_election_op(peer, xml);
+@@ -631,25 +668,7 @@ attrd_peer_message(crm_node_t *peer, xmlNode *xml)
+
+ } else if (pcmk__str_eq(op, PCMK__ATTRD_CMD_SYNC_RESPONSE, pcmk__str_casei)
+ && !pcmk__str_eq(peer->uname, attrd_cluster->uname,
pcmk__str_casei)) {
+- xmlNode *child = NULL;
+-
+- crm_info("Processing %s from %s", op, peer->uname);
+-
+- /* Clear the seen flag for attribute processing held only in the own node. */
+- if (peer_won) {
+- clear_attribute_value_seen();
+- }
+-
+- for (child = pcmk__xml_first_child(xml); child != NULL;
+- child = pcmk__xml_next(child)) {
+- host = crm_element_value(child, PCMK__XA_ATTR_NODE_NAME);
+- attrd_peer_update(peer, child, host, TRUE);
+- }
+-
+- if (peer_won) {
+- /* Synchronize if there is an attribute held only by own node that Writer
does not have. */
+- attrd_current_only_attribute_update(peer, xml);
+- }
++ process_peer_sync_response(peer, peer_won, xml);
+
+ } else if (pcmk__str_eq(op, PCMK__ATTRD_CMD_FLUSH, pcmk__str_casei)) {
+ /* Ignore. The flush command was removed in 2.0.0 but may be
+--
+1.8.3.1
+
+
+From a890a0e5bbbcabf907f51ed0460868035f72464d Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot(a)redhat.com>
+Date: Fri, 11 Jun 2021 14:40:39 -0500
+Subject: [PATCH 08/11] Refactor: pacemaker-attrd: functionize broadcasting
+ local override
+
+... for code isolation
+---
+ daemons/attrd/attrd_commands.c | 42 +++++++++++++++++++++++++++++-------------
+ 1 file changed, 29 insertions(+), 13 deletions(-)
+
+diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c
+index d02d3e6..4783427 100644
+--- a/daemons/attrd/attrd_commands.c
++++ b/daemons/attrd/attrd_commands.c
+@@ -804,6 +804,34 @@ attrd_current_only_attribute_update(crm_node_t *peer, xmlNode *xml)
+ free_xml(sync);
+ }
+
++/*!
++ * \internal
++ * \brief Override an attribute sync with a local value
++ *
++ * Broadcast the local node's value for an attribute that's different from the
++ * value provided in a peer's attribute synchronization response. This ensures a
++ * node's values for itself take precedence and all peers are kept in sync.
++ *
++ * \param[in] a Attribute entry to override
++ *
++ * \return Local instance of attribute value
++ */
++static attribute_value_t *
++broadcast_local_value(attribute_t *a)
++{
++ attribute_value_t *v = g_hash_table_lookup(a->values, attrd_cluster->uname);
++ xmlNode *sync = create_xml_node(NULL, __func__);
++
++ crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE);
++ build_attribute_xml(sync, a->id, a->set, a->uuid, a->timeout_ms,
++ a->user, a->is_private, v->nodename, v->nodeid,
++ v->current, FALSE);
++ attrd_xml_add_writer(sync);
++ send_attrd_message(NULL, sync);
++ free_xml(sync);
++ return v;
++}
++
+ void
+ attrd_peer_update(crm_node_t *peer, xmlNode *xml, const char *host, bool filter)
+ {
+@@ -899,21 +927,9 @@ attrd_peer_update(crm_node_t *peer, xmlNode *xml, const char *host,
bool filter)
+ if (filter && !pcmk__str_eq(v->current, value, pcmk__str_casei)
+ && pcmk__str_eq(host, attrd_cluster->uname, pcmk__str_casei)) {
+
+- xmlNode *sync = create_xml_node(NULL, __func__);
+-
+ crm_notice("%s[%s]: local value '%s' takes priority over
'%s' from %s",
+ attr, host, v->current, value, peer->uname);
+-
+- crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE);
+- v = g_hash_table_lookup(a->values, host);
+- build_attribute_xml(sync, attr, a->set, a->uuid, a->timeout_ms,
a->user,
+- a->is_private, v->nodename, v->nodeid,
v->current, FALSE);
+-
+- attrd_xml_add_writer(sync);
+-
+- /* Broadcast in case any other nodes had the inconsistent value */
+- send_attrd_message(NULL, sync);
+- free_xml(sync);
++ v = broadcast_local_value(a);
+
+ } else if (!pcmk__str_eq(v->current, value, pcmk__str_casei)) {
+ crm_notice("Setting %s[%s]: %s -> %s " CRM_XS " from
%s",
+--
+1.8.3.1
+
+
+From f6f65e3dab070f1bbdf6d1383f4d6173a8840bc9 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot(a)redhat.com>
+Date: Fri, 11 Jun 2021 14:50:29 -0500
+Subject: [PATCH 09/11] Log: pacemaker-attrd: improve messages when
+ broadcasting local-only values
+
+The traces aren't necessary since build_attribute_xml() already logs the same
+info at debug. Also, rename function for clarity, and make static.
+---
+ daemons/attrd/attrd_commands.c | 35 ++++++++++++++++-------------------
+ 1 file changed, 16 insertions(+), 19 deletions(-)
+
+diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c
+index 4783427..356defb 100644
+--- a/daemons/attrd/attrd_commands.c
++++ b/daemons/attrd/attrd_commands.c
+@@ -51,11 +51,12 @@ GHashTable *attributes = NULL;
+
+ void write_attribute(attribute_t *a, bool ignore_delay);
+ void write_or_elect_attribute(attribute_t *a);
+-void attrd_current_only_attribute_update(crm_node_t *peer, xmlNode *xml);
+ void attrd_peer_update(crm_node_t *peer, xmlNode *xml, const char *host, bool filter);
+ void attrd_peer_sync(crm_node_t *peer, xmlNode *xml);
+ void attrd_peer_remove(const char *host, gboolean uncache, const char *source);
+
++static void broadcast_unseen_local_values(crm_node_t *peer, xmlNode *xml);
++
+ static gboolean
+ send_attrd_message(crm_node_t * node, xmlNode * data)
+ {
+@@ -604,7 +605,7 @@ process_peer_sync_response(crm_node_t *peer, bool peer_won, xmlNode
*xml)
+ /* If any attributes are still not marked as seen, the writer doesn't
+ * know about them, so send all peers an update with them.
+ */
+- attrd_current_only_attribute_update(peer, xml);
++ broadcast_unseen_local_values(peer, xml);
+ }
+ }
+
+@@ -768,40 +769,36 @@ attrd_lookup_or_create_value(GHashTable *values, const char *host,
xmlNode *xml)
+ return(v);
+ }
+
+-void
+-attrd_current_only_attribute_update(crm_node_t *peer, xmlNode *xml)
++void
++broadcast_unseen_local_values(crm_node_t *peer, xmlNode *xml)
+ {
+ GHashTableIter aIter;
+ GHashTableIter vIter;
+- attribute_t *a;
++ attribute_t *a = NULL;
+ attribute_value_t *v = NULL;
+- xmlNode *sync = create_xml_node(NULL, __func__);
+- gboolean build = FALSE;
+-
+- crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE);
++ xmlNode *sync = NULL;
+
+ g_hash_table_iter_init(&aIter, attributes);
+ while (g_hash_table_iter_next(&aIter, NULL, (gpointer *) & a)) {
+ g_hash_table_iter_init(&vIter, a->values);
+ while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & v)) {
+- if (pcmk__str_eq(v->nodename, attrd_cluster->uname, pcmk__str_casei)
&& v->seen == FALSE) {
+- crm_trace("Syncing %s[%s] = %s to everyone.(from local only
attributes)", a->id, v->nodename, v->current);
+-
+- build = TRUE;
++ if (!(v->seen) && pcmk__str_eq(v->nodename,
attrd_cluster->uname,
++ pcmk__str_casei)) {
++ if (sync == NULL) {
++ sync = create_xml_node(NULL, __func__);
++ crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE);
++ }
+ build_attribute_xml(sync, a->id, a->set, a->uuid,
a->timeout_ms, a->user, a->is_private,
+ v->nodename, v->nodeid, v->current,
(a->timeout_ms && a->timer ? TRUE : FALSE));
+- } else {
+- crm_trace("Local attribute(%s[%s] = %s) was ignore.(another host) :
[%s]", a->id, v->nodename, v->current, attrd_cluster->uname);
+- continue;
+ }
+ }
+ }
+
+- if (build) {
+- crm_debug("Syncing values to everyone.(from local only attributes)");
++ if (sync != NULL) {
++ crm_debug("Broadcasting local-only values");
+ send_attrd_message(NULL, sync);
++ free_xml(sync);
+ }
+- free_xml(sync);
+ }
+
+ /*!
+--
+1.8.3.1
+
+
+From ab90ffb785ea018556f216b8f540f8c3429a3947 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot(a)redhat.com>
+Date: Fri, 11 Jun 2021 15:04:20 -0500
+Subject: [PATCH 10/11] Refactor: pacemaker-attrd: simplify attribute XML
+ creation function
+
+... and rename for clarity
+---
+ daemons/attrd/attrd_commands.c | 48 ++++++++++++++++++++++++------------------
+ 1 file changed, 27 insertions(+), 21 deletions(-)
+
+diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c
+index 356defb..5b32a77 100644
+--- a/daemons/attrd/attrd_commands.c
++++ b/daemons/attrd/attrd_commands.c
+@@ -125,25 +125,35 @@ cache_remote_node(const char *node_name)
+ CRM_ASSERT(crm_remote_peer_get(node_name) != NULL);
+ }
+
++/*!
++ * \internal
++ * \brief Create an XML representation of an attribute for use in peer messages
++ *
++ * \param[in] parent Create attribute XML as child element of this element
++ * \param[in] a Attribute to represent
++ * \param[in] v Attribute value to represent
++ * \param[in] force_write If true, value should be written even if unchanged
++ *
++ * \return XML representation of attribute
++ */
+ static xmlNode *
+-build_attribute_xml(
+- xmlNode *parent, const char *name, const char *set, const char *uuid, unsigned int
timeout_ms, const char *user,
+- gboolean is_private, const char *peer, uint32_t peerid, const char *value, gboolean
is_force_write)
++add_attribute_value_xml(xmlNode *parent, attribute_t *a, attribute_value_t *v,
++ bool force_write)
+ {
+ xmlNode *xml = create_xml_node(parent, __func__);
+
+- crm_xml_add(xml, PCMK__XA_ATTR_NAME, name);
+- crm_xml_add(xml, PCMK__XA_ATTR_SET, set);
+- crm_xml_add(xml, PCMK__XA_ATTR_UUID, uuid);
+- crm_xml_add(xml, PCMK__XA_ATTR_USER, user);
+- crm_xml_add(xml, PCMK__XA_ATTR_NODE_NAME, peer);
+- if (peerid > 0) {
+- crm_xml_add_int(xml, PCMK__XA_ATTR_NODE_ID, peerid);
++ crm_xml_add(xml, PCMK__XA_ATTR_NAME, a->id);
++ crm_xml_add(xml, PCMK__XA_ATTR_SET, a->set);
++ crm_xml_add(xml, PCMK__XA_ATTR_UUID, a->uuid);
++ crm_xml_add(xml, PCMK__XA_ATTR_USER, a->user);
++ crm_xml_add(xml, PCMK__XA_ATTR_NODE_NAME, v->nodename);
++ if (v->nodeid > 0) {
++ crm_xml_add_int(xml, PCMK__XA_ATTR_NODE_ID, v->nodeid);
+ }
+- crm_xml_add(xml, PCMK__XA_ATTR_VALUE, value);
+- crm_xml_add_int(xml, PCMK__XA_ATTR_DAMPENING, timeout_ms/1000);
+- crm_xml_add_int(xml, PCMK__XA_ATTR_IS_PRIVATE, is_private);
+- crm_xml_add_int(xml, PCMK__XA_ATTR_FORCE, is_force_write);
++ crm_xml_add(xml, PCMK__XA_ATTR_VALUE, v->current);
++ crm_xml_add_int(xml, PCMK__XA_ATTR_DAMPENING, a->timeout_ms / 1000);
++ crm_xml_add_int(xml, PCMK__XA_ATTR_IS_PRIVATE, a->is_private);
++ crm_xml_add_int(xml, PCMK__XA_ATTR_FORCE, force_write);
+
+ return xml;
+ }
+@@ -695,8 +705,7 @@ attrd_peer_sync(crm_node_t *peer, xmlNode *xml)
+ g_hash_table_iter_init(&vIter, a->values);
+ while (g_hash_table_iter_next(&vIter, NULL, (gpointer *) & v)) {
+ crm_debug("Syncing %s[%s] = %s to %s", a->id, v->nodename,
v->current, peer?peer->uname:"everyone");
+- build_attribute_xml(sync, a->id, a->set, a->uuid, a->timeout_ms,
a->user, a->is_private,
+- v->nodename, v->nodeid, v->current, FALSE);
++ add_attribute_value_xml(sync, a, v, false);
+ }
+ }
+
+@@ -788,8 +797,7 @@ broadcast_unseen_local_values(crm_node_t *peer, xmlNode *xml)
+ sync = create_xml_node(NULL, __func__);
+ crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE);
+ }
+- build_attribute_xml(sync, a->id, a->set, a->uuid,
a->timeout_ms, a->user, a->is_private,
+- v->nodename, v->nodeid, v->current,
(a->timeout_ms && a->timer ? TRUE : FALSE));
++ add_attribute_value_xml(sync, a, v, a->timeout_ms &&
a->timer);
+ }
+ }
+ }
+@@ -820,9 +828,7 @@ broadcast_local_value(attribute_t *a)
+ xmlNode *sync = create_xml_node(NULL, __func__);
+
+ crm_xml_add(sync, PCMK__XA_TASK, PCMK__ATTRD_CMD_SYNC_RESPONSE);
+- build_attribute_xml(sync, a->id, a->set, a->uuid, a->timeout_ms,
+- a->user, a->is_private, v->nodename, v->nodeid,
+- v->current, FALSE);
++ add_attribute_value_xml(sync, a, v, false);
+ attrd_xml_add_writer(sync);
+ send_attrd_message(NULL, sync);
+ free_xml(sync);
+--
+1.8.3.1
+
+
+From 540d74130c5c8d9c626d6c50475e4dc4f64234e7 Mon Sep 17 00:00:00 2001
+From: Ken Gaillot <kgaillot(a)redhat.com>
+Date: Fri, 4 Jun 2021 16:34:26 -0500
+Subject: [PATCH 11/11] Fix: pacemaker-attrd: avoid repeated unfencing of
+ remote nodes
+
+The attribute manager can't record a remote node's attributes to the CIB until
+it knows the node is remote. Normally, this is learned when the remote node
+starts, because the controller clears the CRM_OP_PROBED attribute and indicates
+that it is for a remote node.
+
+However, if a cluster node is down when a remote node starts, and later comes
+up, it learns the remote node's existing attributes as part of the attribute
+sync. Previously, this did not include whether each value is for a cluster or
+remote node, so the newly joined attribute manager couldn't write out remote
+nodes' attributes until it learned that via some other event -- which might not
+happen before the node becomes DC, in which case its scheduler will not see any
+unfencing-related node attributes and may wrongly schedule unfencing.
+
+The sync response handling already calls attrd_lookup_or_create_value(), which
+checks PCMK__XA_ATTR_IS_REMOTE, so all we need to do is add that to the sync
+response.
+---
+ daemons/attrd/attrd_commands.c | 6 +++++-
+ 1 file changed, 5 insertions(+), 1 deletion(-)
+
+diff --git a/daemons/attrd/attrd_commands.c b/daemons/attrd/attrd_commands.c
+index 5b32a77..0142383 100644
+--- a/daemons/attrd/attrd_commands.c
++++ b/daemons/attrd/attrd_commands.c
+@@ -43,8 +43,9 @@
+ * 1 1.1.15 PCMK__ATTRD_CMD_UPDATE_BOTH,
+ * PCMK__ATTRD_CMD_UPDATE_DELAY
+ * 2 1.1.17 PCMK__ATTRD_CMD_CLEAR_FAILURE
++ * 3 2.1.1 PCMK__ATTRD_CMD_SYNC_RESPONSE indicates remote nodes
+ */
+-#define ATTRD_PROTOCOL_VERSION "2"
++#define ATTRD_PROTOCOL_VERSION "3"
+
+ int last_cib_op_done = 0;
+ GHashTable *attributes = NULL;
+@@ -150,6 +151,9 @@ add_attribute_value_xml(xmlNode *parent, attribute_t *a,
attribute_value_t *v,
+ if (v->nodeid > 0) {
+ crm_xml_add_int(xml, PCMK__XA_ATTR_NODE_ID, v->nodeid);
+ }
++ if (v->is_remote != 0) {
++ crm_xml_add_int(xml, PCMK__XA_ATTR_IS_REMOTE, 1);
++ }
+ crm_xml_add(xml, PCMK__XA_ATTR_VALUE, v->current);
+ crm_xml_add_int(xml, PCMK__XA_ATTR_DAMPENING, a->timeout_ms / 1000);
+ crm_xml_add_int(xml, PCMK__XA_ATTR_IS_PRIVATE, a->is_private);
+--
+1.8.3.1
+
diff --git a/gating.yaml b/gating.yaml
index 9e15ced..e8aadb9 100644
--- a/gating.yaml
+++ b/gating.yaml
@@ -5,6 +5,7 @@ decision_context: bodhi_update_push_testing
subject_type: koji_build
rules:
- !PassingTestCaseRule {test_case_name: fedora-ci.koji-build.tier0.functional}
+
--- !Policy
product_versions:
- fedora-*
@@ -13,3 +14,9 @@ subject_type: koji_build
rules:
- !PassingTestCaseRule {test_case_name: fedora-ci.koji-build.tier0.functional}
+--- !Policy
+product_versions:
+ - rhel-*
+decision_context: osci_compose_gate
+rules:
+ - !PassingTestCaseRule {test_case_name: osci.brew-build.tier0.functional}
diff --git a/pacemaker.spec b/pacemaker.spec
index 8551cc6..cdfcdb2 100644
--- a/pacemaker.spec
+++ b/pacemaker.spec
@@ -1,9 +1,9 @@
-# Globals and defines to control package behavior (configure these as desired)
+# User-configurable globals and defines to control package behavior
+# (these should not test {with X} values, which are declared later)
## User and group to use for nonprivileged services
%global uname hacluster
%global gname haclient
-%global hacluster_id 189
## Where to install Pacemaker documentation
%if 0%{?rhel}
@@ -17,6 +17,13 @@
## Where bug reports should be submitted
## Leave bug_url undefined to use ClusterLabs default, others define it here
+%if 0%{?rhel}
+%global bug_url
https://bugzilla.redhat.com/
+%else
+%if 0%{?fedora}
+%global bug_url
https://bugz.fedoraproject.org/%{name}
+%endif
+%endif
## What to use as the OCF resource agent root directory
%global ocf_root %{_prefix}/lib/ocf
@@ -25,19 +32,21 @@
## can be incremented to build packages reliably considered "newer"
## than previously built packages with the same pcmkversion)
%global pcmkversion 2.1.0
-%global specversion 3
+%global specversion 6
+
+## Upstream commit (full commit ID, abbreviated commit ID, or tag) to build
+%global commit 7c3f660707a495a1331716ad32cd3ac9d9f8ff58
-## Upstream commit (or git tag, such as "Pacemaker-" plus the
-## {pcmkversion} macro for an official release) to use for this package
-%global commit Pacemaker-2.1.0-rc2
## Since git v2.11, the extent of abbreviation is autoscaled by default
## (used to be constant of 7), so we need to convey it for non-tags, too.
%global commit_abbrev 9
+
# Define conditionals so that "rpmbuild --with <feature>" and
# "rpmbuild --without <feature>" can enable and disable specific
features
-## NOTE: skip --with stonithd
+## Add option to enable support for stonith/external fencing agents
+%bcond_with stonithd
## Add option for whether to support storing sensitive information outside CIB
%if (0%{?fedora} && 0%{?fedora} <= 33) || (0%{?rhel} && 0%{?rhel}
<= 8)
@@ -52,8 +61,13 @@
## Add option to create binaries with coverage analysis
%bcond_with coverage
-## Add option to skip/enable generating documentation
+## Add option to skip (or enable, on RHEL) generating documentation
+## (the build tools aren't available everywhere)
+%if 0%{?rhel}
+%bcond_with doc
+%else
%bcond_without doc
+%endif
## Add option to default to start-up synchronization with SBD.
##
@@ -66,16 +80,19 @@
## Add option to prefix package version with "0."
## (so later "official" packages will be considered updates)
-%bcond_without pre_release
+%bcond_with pre_release
## NOTE: skip --with upstart_job
## Add option to turn off hardening of libraries and daemon executables
%bcond_without hardening
-## Add option to enable links for legacy daemon names
+## Add option to enable (or disable, on RHEL 8) links for legacy daemon names
+%if 0%{?rhel} && 0%{?rhel} <= 8
+%bcond_without legacy_links
+%else
%bcond_with legacy_links
-
+%endif
## Nagios source control identifiers
%global nagios_name nagios-agents-metadata
@@ -94,29 +111,36 @@
## Portion of export/dist tarball name after "pacemaker-", and release version
%if 0%{tag_release}
-%define archive_version %{commit}
+%define archive_version %(c=%{commit}; echo ${c:10})
%define archive_github_url %{commit}#/%{name}-%{archive_version}.tar.gz
-%define pcmk_release %(c=%{commit}; case $c in *-rc[[:digit:]]*%{rparen}
- echo 0.%{specversion}.${c: -3} ;;
- *%{rparen} echo %{specversion} ;; esac)
%else
%define archive_version %(c=%{commit}; echo ${c:0:%{commit_abbrev}})
%define archive_github_url %{archive_version}#/%{name}-%{archive_version}.tar.gz
-%if %{with pre_release}
-%define pcmk_release 0.%{specversion}.%{archive_version}.git
-%else
-%define pcmk_release %{specversion}.%{archive_version}.git
-%endif
%endif
+### Always use a simple release number
+%define pcmk_release %{specversion}
## Base GnuTLS cipher priorities (presumably only the initial, required keyword)
## overridable with "rpmbuild --define 'pcmk_gnutls_priorities
PRIORITY-SPEC'"
%define gnutls_priorities %{?pcmk_gnutls_priorities}%{!?pcmk_gnutls_priorities:@SYSTEM}
+## Different distros name certain packages differently
+## (note: corosync libraries also differ, but all provide corosync-devel)
+%global pkgname_libtool_devel libtool-ltdl-devel
+%global pkgname_libtool_devel_arch libtool-ltdl-devel%{?_isa}
+%global pkgname_bzip2_devel bzip2-devel
+%global pkgname_docbook_xsl docbook-style-xsl
+%global pkgname_gnutls_devel gnutls-devel
+%global pkgname_shadow_utils shadow-utils
+%global pkgname_procps procps-ng
+%global pkgname_glue_libs cluster-glue-libs
+%global pkgname_pcmk_libs %{name}-libs
+%global hacluster_id 189
+
## Distro-specific configuration choices
### Use 2.0-style output when other distro packages don't support current output
-%if 0%{?fedora}
+%if 0%{?fedora} || ( 0%{?rhel} && 0%{?rhel} <= 8 )
%global compat20 --enable-compat-2.0
%endif
@@ -130,6 +154,7 @@
%global resource_stickiness --with-resource-stickiness-default=1
%endif
+
# Python-related definitions
## Turn off auto-compilation of Python files outside Python specific paths,
@@ -145,13 +170,24 @@
sed -e
's!/usr/lib[^[:space:]]*/brp-python-bytecompile[[:space:]].*$!!g'; })
%endif
-## Values that differ by Python major version
+## Prefer Python 3 definitions explicitly, in case 2 is also available
+%if %{defined __python3}
%global python_name python3
-%global python_path
%{?__python3}%{!?__python3:/usr/bin/python%{?python3_pkgversion}%{!?python3_pkgversion:3}}
+%global python_path %{__python3}
%define python_site %{?python3_sitelib}%{!?python3_sitelib:%(
%{python_path} -c 'from distutils.sysconfig import get_python_lib as gpl;
print(gpl(1))' 2>/dev/null)}
-%global python_min 3.2
-# python_min still required?
+%else
+%if %{defined python_version}
+%global python_name python%(echo %{python_version} | cut -d'.' -f1)
+%define python_path %{?__python}%{!?__python:/usr/bin/%{python_name}}
+%else
+%global python_name python
+%global python_path %{?__python}%{!?__python:/usr/bin/python%{?python_pkgversion}}
+%endif
+%define python_site %{?python_sitelib}%{!?python_sitelib:%(
+ %{python_name} -c 'from distutils.sysconfig import get_python_lib as gpl;
print(gpl(1))' 2>/dev/null)}
+%endif
+
# Keep sane profiling data if requested
%if %{with profiling}
@@ -167,23 +203,44 @@ Summary: Scalable High-Availability cluster resource manager
Version: %{pcmkversion}
Release: %{pcmk_release}%{?dist}
License: GPLv2+ and LGPLv2+
-Url:
https://www.clusterlabs.org
-
-# Hint: use "spectool -s 0 pacemaker.spec" (rpmdevtools) to check the final
URL
+Url:
https://www.clusterlabs.org/
+
+# Example:
https://codeload.github.com/ClusterLabs/pacemaker/tar.gz/e91769e
+# will download pacemaker-e91769e.tar.gz
+#
+# The ending part starting with '#' is ignored by github but necessary for
+# rpmbuild to know what the tar archive name is. (The downloaded file will be
+# named correctly only for commit IDs, not tagged releases.)
+#
+# You can use "spectool -s 0 pacemaker.spec" (rpmdevtools) to show final URL.
Source0:
https://codeload.github.com/%{github_owner}/%{name}/tar.gz/%{archive_gith...
Source1:
https://codeload.github.com/%{github_owner}/%{nagios_name}/tar.gz/%{nagio...
-# ---
+
+# upstream commits
+Patch1: 001-ping-agent.patch
+Patch2: 002-pacemakerd-options.patch
+Patch3: 003-pacemakerd-output.patch
+Patch4: 004-check-level.patch
+Patch5: 005-crm_resource.patch
+Patch6: 006-crm_simulate.patch
+Patch7: 007-unfencing-loop.patch
Requires: resource-agents
-Requires: %{name}-libs%{?_isa} = %{version}-%{release}
+Requires: %{pkgname_pcmk_libs}%{?_isa} = %{version}-%{release}
Requires: %{name}-cluster-libs%{?_isa} = %{version}-%{release}
Requires: %{name}-cli = %{version}-%{release}
%{?systemd_requires}
-# Pacemaker requires a minimum Python functionality
-Requires: %{python_name} >= %{python_min}
-BuildRequires: make
-BuildRequires: %{python_name}-devel >= %{python_min}
+%if %{defined centos}
+ExclusiveArch: aarch64 i686 ppc64le s390x x86_64 %{arm}
+%else
+%if 0%{?rhel}
+ExclusiveArch: aarch64 i686 ppc64le s390x x86_64
+%endif
+%endif
+
+Requires: %{python_path}
+BuildRequires: %{python_name}-devel
# Pacemaker requires a minimum libqb functionality
Requires: libqb >= 0.17.0
@@ -192,37 +249,41 @@ BuildRequires: libqb-devel >= 0.17.0
# Required basic build tools
BuildRequires: coreutils findutils grep sed
BuildRequires: autoconf automake gcc make pkgconfig
-BuildRequires: libtool libtool-ltdl-devel
+BuildRequires: libtool %{?pkgname_libtool_devel}
# Required for core functionality
BuildRequires: pkgconfig(glib-2.0) >= 2.42
BuildRequires: libxml2-devel libxslt-devel libuuid-devel
-BuildRequires: bzip2-devel
+BuildRequires: %{pkgname_bzip2_devel}
# Enables optional functionality
-BuildRequires: ncurses-devel docbook-style-xsl
-BuildRequires: help2man gnutls-devel pam-devel pkgconfig(dbus-1)
+BuildRequires: ncurses-devel %{pkgname_docbook_xsl}
+BuildRequires: help2man %{pkgname_gnutls_devel} pam-devel pkgconfig(dbus-1)
BuildRequires: pkgconfig(systemd)
+# RH patches are created by git, so we need git to apply them
+BuildRequires: git
+
Requires: corosync >= 2.0.0
BuildRequires: corosync-devel >= 2.0.0
-#XXX
-#BuildRequires: pkgconfig(libcpg)
-#BuildRequires: pkgconfig(libcfg)
+
+%if %{with stonithd}
+BuildRequires: %{pkgname_glue_libs}-devel
+%endif
%if %{with doc}
BuildRequires: asciidoc inkscape %{python_name}-sphinx
%endif
-# git-style patch application
-#BuildRequires: git
-
Provides: pcmk-cluster-manager = %{version}-%{release}
Provides: pcmk-cluster-manager%{?_isa} = %{version}-%{release}
-# Pacemaker uses the crypto/md5 module from gnulib
+# Bundled bits
+## Pacemaker uses the crypto/md5-buffer module from gnulib
+%if 0%{?fedora} || 0%{?rhel}
Provides: bundled(gnulib)
+%endif
%description
Pacemaker is an advanced, scalable High-Availability cluster resource
@@ -236,18 +297,18 @@ when related resources fail and can be configured to periodically
check
resource health.
Available rpmbuild rebuild options:
- --with(out) : cibsecrets coverage doc hardening pre_release profiling
+ --with(out) : cibsecrets coverage doc hardening pre_release profiling stonithd
%package cli
License: GPLv2+ and LGPLv2+
Summary: Command line tools for controlling Pacemaker clusters
-Requires: %{name}-libs%{?_isa} = %{version}-%{release}
+Requires: %{pkgname_pcmk_libs}%{?_isa} = %{version}-%{release}
Recommends: pcmk-cluster-manager = %{version}-%{release}
# For crm_report
Recommends: tar
Recommends: bzip2
Requires: perl-TimeDate
-Requires: procps-ng
+Requires: %{pkgname_procps}
Requires: psmisc
Requires(post):coreutils
@@ -259,28 +320,29 @@ The %{name}-cli package contains command line tools that can be
used
to query and control the cluster from machines that may, or may not,
be part of the cluster.
-%package libs
+%package -n %{pkgname_pcmk_libs}
License: GPLv2+ and LGPLv2+
Summary: Core Pacemaker libraries
-Requires(pre): shadow-utils
+Requires(pre): %{pkgname_shadow_utils}
Requires: %{name}-schemas = %{version}-%{release}
# sbd 1.4.0+ supports the libpe_status API for pe_working_set_t
# sbd 1.4.2+ supports startup/shutdown handshake via pacemakerd-api
# and handshake defaults to enabled for rhel builds
-# sbd 1.4.3+ handshake defaults to enabled for fedora builds
-Conflicts: sbd < 1.4.3
+# sbd 1.5.0+ handshake defaults to enabled with upstream sbd-release
+# implicitly supports handshake defaults to enabled in this spec
+Conflicts: sbd < 1.5.0
-%description libs
+%description -n %{pkgname_pcmk_libs}
Pacemaker is an advanced, scalable High-Availability cluster resource
manager.
-The %{name}-libs package contains shared libraries needed for cluster
+The %{pkgname_pcmk_libs} package contains shared libraries needed for cluster
nodes and those just running the CLI tools.
%package cluster-libs
License: GPLv2+ and LGPLv2+
Summary: Cluster Libraries used by Pacemaker
-Requires: %{name}-libs%{?_isa} = %{version}-%{release}
+Requires: %{pkgname_pcmk_libs}%{?_isa} = %{version}-%{release}
%description cluster-libs
Pacemaker is an advanced, scalable High-Availability cluster resource
@@ -292,7 +354,7 @@ libraries needed for nodes that will form part of the cluster nodes.
%package remote
License: GPLv2+ and LGPLv2+
Summary: Pacemaker remote executor daemon for non-cluster nodes
-Requires: %{name}-libs%{?_isa} = %{version}-%{release}
+Requires: %{pkgname_pcmk_libs}%{?_isa} = %{version}-%{release}
Requires: %{name}-cli = %{version}-%{release}
Requires: resource-agents
# -remote can be fully independent of systemd
@@ -308,35 +370,40 @@ The %{name}-remote package contains the Pacemaker Remote daemon
which is capable of extending pacemaker functionality to remote
nodes not running the full corosync/cluster stack.
-%package libs-devel
+%package -n %{pkgname_pcmk_libs}-devel
License: GPLv2+ and LGPLv2+
Summary: Pacemaker development package
-Requires: %{name}-libs%{?_isa} = %{version}-%{release}
+Requires: %{pkgname_pcmk_libs}%{?_isa} = %{version}-%{release}
Requires: %{name}-cluster-libs%{?_isa} = %{version}-%{release}
-Requires: libtool-ltdl-devel%{?_isa} libuuid-devel%{?_isa}
+Requires: %{?pkgname_libtool_devel_arch} libuuid-devel%{?_isa}
Requires: libxml2-devel%{?_isa} libxslt-devel%{?_isa}
-Requires: bzip2-devel%{?_isa} glib2-devel%{?_isa}
+Requires: %{pkgname_bzip2_devel}%{?_isa} glib2-devel%{?_isa}
Requires: libqb-devel%{?_isa}
Requires: corosync-devel%{?_isa} >= 2.0.0
-%description libs-devel
+%description -n %{pkgname_pcmk_libs}-devel
Pacemaker is an advanced, scalable High-Availability cluster resource
manager.
-The %{name}-libs-devel package contains headers and shared libraries
+The %{pkgname_pcmk_libs}-devel package contains headers and shared libraries
for developing tools for Pacemaker.
%package cts
License: GPLv2+ and LGPLv2+
Summary: Test framework for cluster-related technologies like Pacemaker
Requires: %{python_path}
-Requires: %{python_name} >= %{python_min}
-Requires: %{name}-libs = %{version}-%{release}
-Requires: procps-ng
+Requires: %{pkgname_pcmk_libs} = %{version}-%{release}
+Requires: %{name}-cli = %{version}-%{release}
+Requires: %{pkgname_procps}
Requires: psmisc
BuildArch: noarch
+# systemd Python bindings are a separate package in some distros
+%if %{defined systemd_requires}
+%if %{defined fedora} || %{defined rhel}
Requires: %{python_name}-systemd
+%endif
+%endif
%description cts
Test framework for cluster-related technologies like Pacemaker
@@ -382,7 +449,10 @@ The metadata files required for Pacemaker to execute the nagios
plugin
monitor resources.
%prep
-%setup -q -a 1 -n %{name}-%{archive_version}
+%autosetup -a 1 -n %{name}-%{archive_version} -S git_am -p 1
+# in f33 s390x complains but shouldn't hurt globally
+# as configure.ac is checking for support
+sed -i configure.ac -e "s/-Wall/-Wall -Wno-format-truncation/"
%build
@@ -420,11 +490,10 @@ export LDFLAGS_HARDENED_LIB="%{?_hardening_ldflags}"
--with-initdir=%{_initrddir} \
--with-runstatedir=%{_rundir} \
--localstatedir=%{_var} \
- --with-version=%{version}-%{release} \
- --with-bug-url=https://bugz.fedoraproject.org/%{name} \
- --with-nagios \
- --with-nagios-metadata-dir=%{_datadir}/pacemaker/nagios/plugins-metadata/ \
- --with-nagios-plugin-dir=%{_libdir}/nagios/plugins/
+ --with-nagios \
+ --with-nagios-metadata-dir=%{_datadir}/pacemaker/nagios/plugins-metadata/ \
+ --with-nagios-plugin-dir=%{_libdir}/nagios/plugins/ \
+ --with-version=%{version}-%{release}
make %{_smp_mflags} V=1
@@ -453,11 +522,7 @@ done
mkdir -p ${RPM_BUILD_ROOT}%{_localstatedir}/lib/rpm-state/%{name}
-# These are not actually scripts
-find %{buildroot} -name '*.xml' -type f -print0 | xargs -0 chmod a-x
-
-# Don't package static libs
-find %{buildroot} -name '*.a' -type f -print0 | xargs -0 rm -f
+# Don't package libtool archives
find %{buildroot} -name '*.la' -type f -print0 | xargs -0 rm -f
# Do not package these either
@@ -475,7 +540,7 @@ rm -f %{buildroot}/%{_initrddir}/pacemaker
rm -f %{buildroot}/%{_initrddir}/pacemaker_remote
# Byte-compile Python sources where suitable and the distro procedures known
-%if %{defined py_byte_compile} && %{defined python_path}
+%if %{defined py_byte_compile}
%{py_byte_compile %{python_path} %{buildroot}%{_datadir}/pacemaker/tests}
%if !%{defined _python_bytecompile_extra}
%{py_byte_compile %{python_path} %{buildroot}%{python_site}/cts}
@@ -552,15 +617,14 @@ fi
%postun cli
%systemd_postun_with_restart crm_mon.service
-%pre libs
-# XXX keep an eye on
https://fedoraproject.org/wiki/Changes/SystemdSysusers
-# reopened recently:
-#
https://lists.fedoraproject.org/archives/list/devel@lists.fedoraproject.o...
+%pre -n %{pkgname_pcmk_libs}
+# @TODO Use sysusers.d:
+#
https://fedoraproject.org/wiki/Changes/Adopting_sysusers.d_format
getent group %{gname} >/dev/null || groupadd -r %{gname} -g %{hacluster_id}
getent passwd %{uname} >/dev/null || useradd -r -g %{gname} -u %{hacluster_id} -s
/sbin/nologin -c "cluster user" %{uname}
exit 0
-%ldconfig_scriptlets libs
+%ldconfig_scriptlets -n %{pkgname_pcmk_libs}
%ldconfig_scriptlets cluster-libs
%files
@@ -667,7 +731,7 @@ exit 0
%dir %attr (770, %{uname}, %{gname}) %{_var}/log/pacemaker
%dir %attr (770, %{uname}, %{gname}) %{_var}/log/pacemaker/bundles
-%files libs
+%files -n %{pkgname_pcmk_libs}
%{_libdir}/libcib.so.*
%{_libdir}/liblrmd.so.*
%{_libdir}/libcrmservice.so.*
@@ -716,7 +780,7 @@ exit 0
%doc COPYING
%doc ChangeLog
-%files libs-devel
+%files -n %{pkgname_pcmk_libs}-devel
%{_includedir}/pacemaker
%{_libdir}/*.so
%if %{with coverage}
@@ -742,7 +806,24 @@ exit 0
%license %{nagios_name}-%{nagios_hash}/COPYING
%changelog
-* Tue Jun 1 2021 Klaus Wenninger <kwenning(a)redhat.com> - 2.1.0-0.3.rc2
+* Mon Jul 5 2021 Klaus Wenninger <kwenning(a)redhat.com> - 2.1.0-6
+- synced/merged with CS9 spec-file for current 2.1.0-release build
+
+* Tue Jun 8 2021 Klaus Wenninger <kwenning(a)redhat.com> - 2.1.0-0.5.rc3
+- silence f33 s390x build complaining about possible format-trucation
+
+* Mon Jun 07 2021 Python Maint <python-maint(a)redhat.com> - 2.1.0-0.4.rc3.1
+- Rebuilt for Python 3.10
+
+* Mon Jun 07 2021 Klaus Wenninger <kwenning(a)redhat.com> - 2.1.0-0.4.rc3
+- Update for new upstream tarball for release candidate: Pacemaker-2.1.0-rc3,
+ for full details, see included ChangeLog file or
+
https://github.com/ClusterLabs/pacemaker/releases/tag/Pacemaker-2.1.0-rc3
+
+* Fri Jun 04 2021 Python Maint <python-maint(a)redhat.com> - 2.1.0-0.3.rc2.1
+- Rebuilt for Python 3.10
+
+* Mon May 31 2021 Klaus Wenninger <kwenning(a)redhat.com> - 2.1.0-0.3.rc2
- Update for new upstream tarball for release candidate: Pacemaker-2.1.0-rc2,
for full details, see included ChangeLog file or
https://github.com/ClusterLabs/pacemaker/releases/tag/Pacemaker-2.1.0-rc2
diff --git a/sources b/sources
index 76e3c9a..2ab7f32 100644
--- a/sources
+++ b/sources
@@ -1,2 +1,2 @@
SHA512 (nagios-agents-metadata-105ab8a7b2c16b9a29cf1c1596b80136eeef332b.tar.gz) =
11ddeb48a4929e7642b6dfa9c7962aa1d7a1af1c569830f55ed6cd6773abac13377317327bc1db8411c8077884f83f81cc54d746c834b63a99fa6dc219b5caad
-SHA512 (pacemaker-Pacemaker-2.1.0-rc2.tar.gz) =
01396cd23b0cb49cab40b870e227b132f1e0c9b94cf6ac29d58e61bd3ff4ee4ea95fa6509c9a4a09179284b8fdbdba8319ceb46e8f94ce9cea92cf13832f9f4d
+SHA512 (pacemaker-7c3f66070.tar.gz) =
e1c53d019c965395affc1bd78358927617c9bc8572c405ee884737a8213455c5e85102e6f58bb5239ab23ebe5aa794f72c9cc0746900b7a6cc14fd3cf1879064