>From d6139d1b5fd5d566d3e07d0e48911b306e8e42ca Mon Sep 17 00:00:00 2001
From: Martin Nagy <mnagy@redhat.com>
Date: Thu, 29 Oct 2009 22:31:59 +0100
Subject: [PATCH 3/3] Add fail over utility functions

These functions should be used by providers to centrally manage lists of
servers. Servers are grouped into services and each service has it's own
list of servers. If, however, you will try to add a same server into two
different services, they will share a common structure. This means that
a host will only be resolved once.
---
 server/Makefile.am             |   20 ++-
 server/providers/fail_over.c   |  517 ++++++++++++++++++++++++++++++++++++++++
 server/providers/fail_over.h   |  110 +++++++++
 server/tests/fail_over-tests.c |  295 +++++++++++++++++++++++
 4 files changed, 941 insertions(+), 1 deletions(-)
 create mode 100644 server/providers/fail_over.c
 create mode 100644 server/providers/fail_over.h
 create mode 100644 server/tests/fail_over-tests.c

diff --git a/server/Makefile.am b/server/Makefile.am
index afb659f..2be22b0 100644
--- a/server/Makefile.am
+++ b/server/Makefile.am
@@ -69,7 +69,8 @@ if HAVE_CHECK
         krb5-utils-tests \
         check_and_open-tests \
         files-tests \
-        refcount-tests
+        refcount-tests \
+        fail_over-tests
 endif
 
 check_PROGRAMS = \
@@ -207,6 +208,9 @@ if BUILD_ARES_PARSE_TXT
     SSSD_RESOLV_OBJ += resolv/ares/ares_parse_txt_reply.c
 endif
 
+SSSD_FAILOVER_OBJ = \
+    providers/fail_over.c \
+    $(SSSD_RESOLV_OBJ)
 
 SSSD_LIBS = \
     $(TALLOC_LIBS) \
@@ -268,6 +272,7 @@ dist_noinst_HEADERS = \
     confdb/confdb_setup.h \
     providers/data_provider.h \
     providers/dp_backend.h \
+    providers/fail_over.h \
     providers/providers.h \
     providers/krb5/krb5_auth.h \
     providers/krb5/krb5_common.h \
@@ -467,6 +472,19 @@ refcount_tests_CFLAGS = \
 refcount_tests_LDADD = \
     $(SSSD_LIBS) \
     $(CHECK_LIBS)
+
+fail_over_tests_SOURCES = \
+    tests/fail_over-tests.c \
+    tests/common.c \
+    $(SSSD_FAILOVER_OBJ) \
+    $(CHECK_OBJ) \
+    $(SSSD_UTIL_OBJ)
+fail_over_tests_CFLAGS = \
+    $(CHECK_CFLAGS)
+fail_over_tests_LDADD = \
+    $(SSSD_LIBS) \
+    $(CHECK_LIBS) \
+    $(CARES_LIBS)
 endif
 
 ####################
diff --git a/server/providers/fail_over.c b/server/providers/fail_over.c
new file mode 100644
index 0000000..5ae0c2c
--- /dev/null
+++ b/server/providers/fail_over.c
@@ -0,0 +1,517 @@
+/*
+   SSSD
+
+   Fail over helper functions.
+
+   Authors:
+        Martin Nagy <mnagy@redhat.com>
+
+   Copyright (C) Red Hat, Inc 2009
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <sys/time.h>
+
+#include <errno.h>
+#include <stdbool.h>
+#include <strings.h>
+#include <talloc.h>
+
+#include "util/dlinklist.h"
+#include "util/refcount.h"
+#include "util/util.h"
+#include "providers/fail_over.h"
+#include "resolv/async_resolv.h"
+
+#define STATUS_DIFF(p, tv2) ((p)->last_status_change.tv_sec - (tv2).tv_sec)
+
+#define DEFAULT_SERVICE_STATUS SERVICE_NEUTRAL
+#define DEFAULT_SERVER_STATUS SERVER_NAME_NOT_RESOLVED
+
+static struct {
+    int initialized;
+    struct fo_service *service_list;
+    struct server_common *server_common_list;
+
+    /* Settings. */
+    time_t retry_time;
+} global = {
+    0, NULL, NULL, 5 * 60
+};
+
+struct fo_service {
+    struct fo_service *prev;
+    struct fo_service *next;
+
+    char *name;
+    int default_port;
+    struct fo_server *active_server;
+    struct fo_server *server_list;
+};
+
+struct fo_server {
+    struct fo_server *prev;
+    struct fo_server *next;
+
+    void *user_data;
+    int port;
+    int service_status;
+    struct fo_service *service;
+    struct timeval last_status_change;
+    struct server_common *common;
+};
+
+struct server_common {
+    REFCOUNT_COMMON;
+
+    struct server_common *prev;
+    struct server_common *next;
+
+    char *name;
+    struct hostent *hostent;
+    struct get_server_request *request_list;
+    int server_status;
+    struct timeval last_status_change;
+};
+
+struct get_server_request {
+    struct get_server_request *prev;
+    struct get_server_request *next;
+
+    struct server_common *server_common;
+    struct tevent_req *req;
+    struct tevent_context *ev;
+};
+
+struct status {
+    int value;
+    struct timeval last_change;
+};
+
+void
+fo_set_retry_time(time_t retry_time)
+{
+    global.retry_time = retry_time;
+}
+
+/*
+ * This function will return the status of the server. If the status was
+ * last updated a long time ago, we will first reset the status.
+ */
+static enum server_status
+get_server_status(struct fo_server *server)
+{
+    struct timeval tv;
+
+    gettimeofday(&tv, NULL);
+    if (server->common->server_status == SERVER_NOT_WORKING
+        && STATUS_DIFF(server->common, tv) > global.retry_time
+        && global.retry_time > 0) {
+        server->common->server_status = SERVER_NAME_RESOLVED;
+        server->last_status_change.tv_sec = tv.tv_sec;
+    }
+
+    return server->common->server_status;
+}
+
+/*
+ * This function will return the status of the service. If the status was
+ * last updated a long time ago, we will first reset the status.
+ */
+static enum service_status
+get_service_status(struct fo_server *server)
+{
+    struct timeval tv;
+
+    gettimeofday(&tv, NULL);
+    if (server->service_status == SERVICE_NOT_WORKING
+        && STATUS_DIFF(server, tv) > global.retry_time
+        && global.retry_time > 0) {
+        server->service_status = SERVICE_NEUTRAL;
+        server->last_status_change.tv_sec = tv.tv_sec;
+    }
+
+    return server->service_status;
+}
+
+static int
+server_works(struct fo_server *server)
+{
+    if (get_server_status(server) == SERVER_NOT_WORKING)
+        return 0;
+
+    return 1;
+}
+
+static int
+service_works(struct fo_server *server)
+{
+    if (!server_works(server))
+        return 0;
+    if (get_service_status(server) == SERVICE_NOT_WORKING)
+        return 0;
+
+    return 1;
+}
+
+static int
+service_destructor(struct fo_service *service)
+{
+    DLIST_REMOVE(global.service_list, service);
+    return 0;
+}
+
+int
+fo_new_service(TALLOC_CTX *mem_ctx, const char *name, int port,
+               struct fo_service **_service)
+{
+    struct fo_service *service;
+    int ret;
+
+    ret = fo_get_service(name, &service);
+    if (ret == EOK) {
+        DEBUG(1, ("Service %s already exists\n", name));
+        return EEXIST;
+    } else if (ret != ENOENT) {
+        return ret;
+    }
+
+    service = talloc_zero(mem_ctx, struct fo_service);
+    if (service == NULL)
+        return ENOMEM;
+
+    service->name = talloc_strdup(service, name);
+    if (service->name == NULL) {
+        talloc_free(service);
+        return ENOMEM;
+    }
+
+    service->default_port = port;
+    DLIST_ADD(global.service_list, service);
+
+    talloc_set_destructor(service, service_destructor);
+    *_service = service;
+
+    return EOK;
+}
+
+int
+fo_get_service(const char *name, struct fo_service **_service)
+{
+    struct fo_service *service;
+
+    DLIST_FOR_EACH(service, global.service_list) {
+        if (!strcasecmp(name, service->name)) {
+            *_service = service;
+            return EOK;
+        }
+    }
+
+    return ENOENT;
+}
+
+static int
+get_server_common(TALLOC_CTX *mem_ctx, const char *name,
+                  struct server_common **_common)
+{
+    struct server_common *common;
+
+    DLIST_FOR_EACH(common, global.server_common_list) {
+        if (!strcmp(name, common->name)) {
+            *_common = rc_reference(mem_ctx, struct server_common, common);
+            if (_common == NULL)
+                return ENOMEM;
+            return EOK;
+        }
+    }
+
+    return ENOENT;
+}
+
+static struct server_common *
+create_server_common(TALLOC_CTX *mem_ctx, const char *name)
+{
+    struct server_common *common;
+
+    common = rc_alloc(mem_ctx, struct server_common);
+    if (common == NULL)
+        return NULL;
+
+    common->name = talloc_strdup(common, name);
+    if (common->name == NULL) {
+        talloc_free(common);
+        return NULL;
+    }
+
+    common->prev = NULL;
+    common->next = NULL;
+    common->hostent = NULL;
+    common->request_list = NULL;
+    common->server_status = DEFAULT_SERVER_STATUS;
+    common->last_status_change.tv_sec = 0;
+    common->last_status_change.tv_usec = 0;
+
+    DLIST_ADD_END(global.server_common_list, common, struct server_common *);
+    return common;
+}
+
+int
+fo_add_server(struct fo_service *service, const char *name, int port,
+              bool enable_resolving)
+{
+    struct fo_server *server;
+    int ret;
+
+    server = talloc_zero(service, struct fo_server);
+    if (server == NULL)
+        return ENOMEM;
+
+    ret = get_server_common(server, name, &server->common);
+    if (ret == ENOENT) {
+        server->common = create_server_common(server, name);
+        if (server->common == NULL) {
+            talloc_free(server);
+            return ENOMEM;
+        }
+    } else if (ret != EOK) {
+        return ret;
+    }
+
+    if (!enable_resolving) {
+        if (server->common->hostent != NULL)
+            talloc_free(server->common->hostent);
+        fo_set_server_status(server, SERVER_NAME_RESOLVED);
+    }
+
+    if (port == FO_DEFAULT_PORT)
+        server->port = service->default_port;
+    else
+        server->port = port;
+
+    server->service = service;
+    server->service_status = DEFAULT_SERVICE_STATUS;
+
+    DLIST_ADD_END(service->server_list, server, struct fo_server *);
+
+    return EOK;
+}
+
+static int
+get_first_server_entity(struct fo_service *service, struct fo_server **_server)
+{
+    struct fo_server *server;
+
+    /* If we already have a working server, use that one. */
+    server = service->active_server;
+    if (server != NULL) {
+        if (service_works(server)) {
+            *_server = server;
+            return EOK;
+        }
+        service->active_server = NULL;
+    }
+
+    /* Otherwise iterate through the server list. */
+    DLIST_FOR_EACH(server, service->server_list) {
+        if (service_works(server)) {
+            *_server = server;
+            return EOK;
+        }
+    }
+
+    return ENOENT;
+}
+
+static int
+set_lookup_hook(TALLOC_CTX *mem_ctx, struct fo_server *server,
+                struct tevent_req *req, struct tevent_context *ev)
+{
+    struct get_server_request *request;
+
+    request = talloc(mem_ctx, struct get_server_request);
+    if (request == NULL)
+        goto nomem;
+    request->server_common = server->common;
+    request->req = req;
+    request->ev = ev;
+    DLIST_ADD(server->common->request_list, request);
+
+    return EOK;
+nomem:
+    DEBUG(1, ("No memory\n"));
+    talloc_free(request);
+    return ENOMEM;
+}
+
+/*******************************************************************
+ * Get server to connect to.                                       *
+ *******************************************************************/
+
+struct get_server_state {
+    struct fo_server *server;
+};
+
+static void fo_get_server_done(struct tevent_req *subreq);
+
+struct tevent_req *
+fo_get_server_send(TALLOC_CTX *mem_ctx, struct tevent_context *ev,
+                   struct resolv_ctx *resolv, struct fo_service *service)
+{
+    int ret;
+    struct fo_server *server;
+    struct tevent_req *req;
+    struct tevent_req *subreq;
+    struct get_server_state *state;
+
+    req = tevent_req_create(mem_ctx, &state, struct get_server_state);
+    if (req == NULL)
+        return NULL;
+
+    ret = get_first_server_entity(service, &server);
+    if (ret != EOK) {
+        DEBUG(1, ("No available servers for service '%s'\n", service->name));
+
+        tevent_req_error(req, ENOENT);
+        tevent_req_post(req, ev);
+        return req;
+    }
+
+    state->server = server;
+
+    switch (get_server_status(server)) {
+    case SERVER_NAME_NOT_RESOLVED: /* Request name resolution. */
+        /* TODO: Enable IPv6. */
+        subreq = resolv_gethostbyname_send(req, ev, resolv,
+                                           server->common->name, AF_INET);
+        if (subreq == NULL)
+            goto done;
+        tevent_req_set_callback(subreq, fo_get_server_done, req);
+        break;
+    case SERVER_RESOLVING_NAME:
+        /* Name resolution is already under way. Just add ourselves into the
+         * waiting queue so we get notified after the operation is finished. */
+        ret = set_lookup_hook(state, server, req, ev);
+        if (ret != EOK)
+            goto done;
+        break;
+    default: /* The name is already resolved. Return immediately. */
+        tevent_req_done(req);
+        tevent_req_post(req, ev);
+        break;
+    }
+
+    return req;
+
+done:
+    talloc_free(req);
+    return NULL;
+}
+
+static void
+fo_get_server_done(struct tevent_req *subreq)
+{
+    int resolv_status;
+    struct tevent_req *req;
+    struct get_server_state *state;
+    struct get_server_request *request;
+    struct get_server_request *next_request;
+    struct server_common *common;
+
+    req = tevent_req_callback_data(subreq, struct tevent_req);
+    state = tevent_req_data(req, struct get_server_state);
+
+    common = state->server->common;
+    if (common->hostent != NULL)
+        talloc_free(common->hostent);
+    resolv_gethostbyname_recv(common, subreq, &resolv_status, NULL,
+                              &common->hostent);
+    talloc_free(subreq);
+
+    /* Take care of all requests for this server. */
+    for (request = common->request_list; request; request = next_request) {
+        next_request = request->next;
+
+        DLIST_REMOVE(common->request_list, request);
+        if (resolv_status)
+            tevent_req_error(request->req, resolv_status);
+        else
+            tevent_req_done(request->req);
+    }
+
+    if (resolv_status)
+        tevent_req_error(req, resolv_status);
+    else
+        tevent_req_done(req);
+}
+
+int
+fo_get_server_recv(struct tevent_req *req, struct fo_server **server)
+{
+    struct get_server_state *state;
+    enum tevent_req_state tstate;
+    uint64_t err = EIO;
+
+    state = tevent_req_data(req, struct get_server_state);
+
+    if (tevent_req_is_error(req, &tstate, &err))
+        return err;
+
+    if (server)
+        *server = state->server;
+
+    return EOK;
+}
+
+void
+fo_set_server_status(struct fo_server *server, enum server_status status)
+{
+    server->common->server_status = status;
+    gettimeofday(&server->common->last_status_change, NULL);
+}
+
+void
+fo_set_service_status(struct fo_server *server, enum service_status status)
+{
+    server->service_status = status;
+    gettimeofday(&server->last_status_change, NULL);
+    if (status == SERVICE_WORKING) {
+        fo_set_server_status(server, SERVER_WORKING);
+        server->service->active_server = server;
+    }
+}
+
+void
+fo_set_server_user_data(struct fo_server *server, void *user_data)
+{
+    server->user_data = user_data;
+}
+
+void *
+fo_get_server_user_data(struct fo_server *server)
+{
+    return server->user_data;
+}
+
+int
+fo_get_server_port(struct fo_server *server)
+{
+    return server->port;
+}
+
+struct hostent *
+fo_get_server_hostent(struct fo_server *server)
+{
+    return server->common->hostent;
+}
diff --git a/server/providers/fail_over.h b/server/providers/fail_over.h
new file mode 100644
index 0000000..103bcc0
--- /dev/null
+++ b/server/providers/fail_over.h
@@ -0,0 +1,110 @@
+#ifndef __FAIL_OVER_H__
+#define __FAIL_OVER_H__
+
+#include <stdbool.h>
+#include <talloc.h>
+
+/* Some forward declarations that don't have to do anything with fail over. */
+struct hostent;
+struct resolv_ctx;
+struct tevent_context;
+struct tevent_req;
+
+/* Use service default port. */
+#define FO_DEFAULT_PORT (-1)
+
+enum service_status {
+    SERVICE_NEUTRAL,    /* We didn't try this service yet. */
+    SERVICE_WORKING,    /* This service was reported to work. */
+    SERVICE_NOT_WORKING /* This service was reported to not work. */
+};
+
+enum server_status {
+    SERVER_NAME_NOT_RESOLVED, /* We didn't yet resolved the host name. */
+    SERVER_RESOLVING_NAME,    /* Name resolving is in progress. */
+    SERVER_NAME_RESOLVED,     /* We resolved the host name but didn't try to connect. */
+    SERVER_WORKING,           /* We successfully connected to the server. */
+    SERVER_NOT_WORKING        /* We tried and failed to connect to the server. */
+};
+
+struct fo_service;
+struct fo_server;
+
+/*
+ * Set time (in seconds) after which a server that is marked as not working
+ * will be considered again.
+ */
+void fo_set_retry_time(time_t retry_time);
+
+/*
+ * Create a new service structure, saving it to the location pointed to by
+ * '_service'. The needed memory will be allocated from 'mem_ctx'. Service
+ * name will be set to 'name'. The default port used for servers will be
+ * 'port'.
+ */
+int fo_new_service(TALLOC_CTX *mem_ctx,
+                   const char *name,
+                   int port,
+                   struct fo_service **_service);
+
+/*
+ * Look up service named 'name' from the global list. Target of '_service' will
+ * be set to the service if it was found.
+ */
+int fo_get_service(const char *name,
+                   struct fo_service **_service);
+
+/*
+ * Adds a server 'name' to the 'service'. Port 'port' will be used for
+ * connection. If 'port' is FO_DEFAULT_PORT, the default port of 'service' will
+ * be used instead.
+ */
+int fo_add_server(struct fo_service *service,
+                  const char *name,
+                  int port,
+                  bool enable_resolving);
+
+/*
+ * Request the first server from the service's list of servers. It is only
+ * considered if it is not marked as not working (or the retry interval already
+ * passed). If the server address wasn't resolved yet, it will be done.
+ */
+struct tevent_req *fo_get_server_send(TALLOC_CTX *mem_ctx,
+                                      struct tevent_context *ev,
+                                      struct resolv_ctx *resolv,
+                                      struct fo_service *service);
+
+int fo_get_server_recv(struct tevent_req *req,
+                       struct fo_server **server);
+
+/*
+ * Set feedback about 'server'. Caller should use this to indicate a problem
+ * with the server itself, not only with the service on that server. This
+ * should be used, for example, when the IP address of the server can't be
+ * reached. This setting can affect other services as well, since they can
+ * share the same server.
+ */
+void fo_set_server_status(struct fo_server *server,
+                          enum server_status status);
+
+/*
+ * Set feedback about the service status. This function should be used when
+ * the server itself is working but the service is not. When status is set
+ * to SERVICE_WORKING, 'server' is also marked as an "active server" for it's
+ * service. When the next fo_get_server_send() function is called, this server
+ * will be preferred. This will hold as long as it is not marked as
+ * not-working.
+ */
+void fo_set_service_status(struct fo_server *server,
+                           enum service_status status);
+
+void fo_set_server_user_data(struct fo_server *server,
+                             void *user_data);
+
+void *fo_get_server_user_data(struct fo_server *server);
+
+int fo_get_server_port(struct fo_server *server);
+
+struct hostent *fo_get_server_hostent(struct fo_server *server);
+
+#endif /* !__FAIL_OVER_H__ */
diff --git a/server/tests/fail_over-tests.c b/server/tests/fail_over-tests.c
new file mode 100644
index 0000000..d4c0fe8
--- /dev/null
+++ b/server/tests/fail_over-tests.c
@@ -0,0 +1,295 @@
+/*
+   SSSD
+
+   Fail over tests.
+
+   Authors:
+        Martin Nagy <mnagy@redhat.com>
+
+   Copyright (C) Red Hat, Inc 2009
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 3 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program.  If not, see <http://www.gnu.org/licenses/>.
+*/
+
+#include <arpa/inet.h>
+
+#include <check.h>
+#include <popt.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <talloc.h>
+#include <tevent.h>
+
+#include "resolv/async_resolv.h"
+#include "tests/common.h"
+#include "util/util.h"
+
+/* Interface under test */
+#include "providers/fail_over.h"
+
+int use_net_test;
+
+struct test_ctx {
+    struct tevent_context *ev;
+    struct resolv_ctx *resolv;
+    int tasks;
+};
+
+struct task {
+    struct test_ctx *test_ctx;
+    const char *location;
+    int recv;
+    int port;
+    int new_server_status;
+    int new_service_status;
+};
+
+static struct test_ctx *
+setup_test(void)
+{
+    struct test_ctx *ctx;
+    int ret;
+
+    ctx = talloc_zero(NULL, struct test_ctx);
+    fail_if(ctx == NULL, "Could not allocate memory for test context");
+
+    ctx->ev = tevent_context_init(ctx);
+    if (ctx->ev == NULL) {
+        talloc_free(ctx);
+        fail("Could not init tevent context");
+    }
+
+    ret = resolv_init(ctx, ctx->ev, &ctx->resolv);
+    if (ret != EOK) {
+        talloc_free(ctx);
+        fail("Could not init resolv context");
+    }
+
+    return ctx;
+}
+
+static void
+test_loop(struct test_ctx *data)
+{
+    while (data->tasks != 0)
+        tevent_loop_once(data->ev);
+}
+
+START_TEST(test_fo_new_service)
+{
+    int i;
+    int ret;
+    void *tmp_ctx;
+    struct fo_service *service;
+    struct fo_service *services[10];
+
+    tmp_ctx = talloc_new(NULL);
+    check_leaks_push(tmp_ctx);
+
+    for (i = 0; i < 10; i++) {
+        char buf[16];
+        sprintf(buf, "service_%d", i);
+
+        check_leaks_push(tmp_ctx);
+        ret = fo_new_service(tmp_ctx, buf, i, &services[i]);
+        fail_if(ret != EOK);
+    }
+
+    ret = fo_new_service(tmp_ctx, "service_3", 1, &service);
+    fail_if(ret != EEXIST);
+
+    for (i = 9; i >= 0; i--) {
+        char buf[16];
+        sprintf(buf, "service_%d", i);
+
+        ret = fo_get_service(buf, &service);
+        fail_if(ret != EOK);
+        fail_if(service != services[i]);
+        talloc_free(service);
+        check_leaks_pop(tmp_ctx);
+
+        ret = fo_get_service(buf, &service);
+        fail_if(ret != ENOENT);
+    }
+
+    check_leaks_pop(tmp_ctx);
+    talloc_free(tmp_ctx);
+}
+END_TEST
+
+static void
+test_get_server_callback(struct tevent_req *req)
+{
+    uint64_t recv_status;
+    int port;
+    struct task *task;
+    struct fo_server *server = NULL;
+    struct hostent *he;
+    int i;
+
+    task = tevent_req_callback_data(req, struct task);
+
+    task->test_ctx->tasks--;
+
+    recv_status = fo_get_server_recv(req, &server);
+    talloc_free(req);
+    fail_if(recv_status != task->recv, "%s: Expected return of %d, got %d",
+            task->location, task->recv, recv_status);
+    if (recv_status != EOK)
+        return;
+    fail_if(server == NULL);
+    port = fo_get_server_port(server);
+    fail_if(port != task->port, "%s: Expected port %d, got %d", task->location,
+            task->port, port);
+
+    if (task->new_service_status >= 0)
+        fo_set_service_status(server, task->new_service_status);
+    if (task->new_server_status >= 0)
+        fo_set_server_status(server, task->new_server_status);
+
+    he = fo_get_server_hostent(server);
+    fail_if(he == NULL, "%s: fo_get_server_hostent() returned NULL");
+    for (i = 0; he->h_addr_list[i]; i++) {
+        char buf[256];
+
+        inet_ntop(he->h_addrtype, he->h_addr_list[i], buf, sizeof(buf));
+        fail_if(strcmp(buf, "127.0.0.1") != 0 && strcmp(buf, "::1") != 0);
+    }
+
+}
+
+#define get_request(a, b, c, d, e, f) \
+       _get_request(a, b, c, d, e, f, __location__)
+
+static void
+_get_request(struct test_ctx *test_ctx, struct fo_service *service,
+             int expected_recv, int expected_port, int new_service_status,
+             int new_server_status, const char *location)
+{
+    struct tevent_req *req;
+    struct task *task;
+
+    task = talloc(test_ctx, struct task);
+    fail_if(task == NULL);
+
+    task->test_ctx = test_ctx;
+    task->recv = expected_recv;
+    task->port = expected_port;
+    task->new_service_status = new_service_status;
+    task->new_server_status = new_server_status;
+    task->location = location;
+    test_ctx->tasks++;
+
+    req = fo_get_server_send(test_ctx, test_ctx->ev, test_ctx->resolv, service);
+    fail_if(req == NULL, "%s: fo_get_server_send() failed", location);
+
+    tevent_req_set_callback(req, test_get_server_callback, task);
+    test_loop(test_ctx);
+}
+
+START_TEST(test_fo_get_server)
+{
+    struct test_ctx *ctx;
+    struct fo_service *service[2];
+
+    ctx = setup_test();
+    fail_if(ctx == NULL);
+
+    /* Add service. */
+    fail_if(fo_new_service(ctx, "http", 80, &service[0]) != EOK);
+
+    fail_if(fo_new_service(ctx, "ldap", 389, &service[1]) != EOK);
+
+    /* Add servers. */
+    fail_if(fo_add_server(service[0], "localhost", 20, true) != EOK);
+    fail_if(fo_add_server(service[0], "127.0.0.1", FO_DEFAULT_PORT, true) != EOK);
+
+    fail_if(fo_add_server(service[1], "localhost", 30, true) != EOK);
+    fail_if(fo_add_server(service[1], "127.0.0.1", FO_DEFAULT_PORT, true) != EOK);
+
+    /* Make requests. */
+    get_request(ctx, service[0], EOK, 20, SERVICE_WORKING, -1);
+    get_request(ctx, service[0], EOK, 20, -1, SERVER_NOT_WORKING);
+    get_request(ctx, service[0], EOK, 80, SERVICE_WORKING, -1);
+    get_request(ctx, service[0], EOK, 80, SERVICE_NOT_WORKING, -1);
+    get_request(ctx, service[0], ENOENT, 0, -1, -1);
+
+    get_request(ctx, service[1], EOK, 389, SERVICE_WORKING, -1);
+    get_request(ctx, service[1], EOK, 389, -1, SERVER_NOT_WORKING);
+    get_request(ctx, service[1], ENOENT, 0, -1, -1);
+
+    talloc_free(ctx);
+}
+END_TEST
+
+Suite *
+create_suite(void)
+{
+    Suite *s = suite_create("fail_over");
+
+    TCase *tc = tcase_create("FAIL_OVER Tests");
+
+    tcase_add_checked_fixture(tc, leak_check_setup, leak_check_teardown);
+    /* Do some testing */
+    tcase_add_test(tc, test_fo_new_service);
+    tcase_add_test(tc, test_fo_get_server);
+    if (use_net_test) {
+    }
+    /* Add all test cases to the test suite */
+    suite_add_tcase(s, tc);
+
+    return s;
+}
+
+int
+main(int argc, const char *argv[])
+{
+    int opt;
+    poptContext pc;
+    int failure_count;
+    Suite *suite;
+    SRunner *sr;
+    int debug = 0;
+
+    struct poptOption long_options[] = {
+        POPT_AUTOHELP
+        { "debug-level", 'd', POPT_ARG_INT, &debug, 0, "Set debug level", NULL },
+        { "use-net-test", 'n', POPT_ARG_NONE, 0, 'n', "Run tests that need an active internet connection", NULL },
+        POPT_TABLEEND
+    };
+
+    pc = poptGetContext(argv[0], argc, argv, long_options, 0);
+    while((opt = poptGetNextOpt(pc)) != -1) {
+        switch(opt) {
+        case 'n':
+            use_net_test = 1;
+            break;
+
+        default:
+            fprintf(stderr, "\nInvalid option %s: %s\n\n",
+                    poptBadOption(pc, 0), poptStrerror(opt));
+            poptPrintUsage(pc, stderr, 0);
+            return 1;
+        }
+    }
+    poptFreeContext(pc);
+    debug_level = debug;
+
+    suite = create_suite();
+    sr = srunner_create(suite);
+    srunner_run_all(sr, CK_VERBOSE);
+    failure_count = srunner_ntests_failed(sr);
+    srunner_free(sr);
+    return (failure_count == 0 ? EXIT_SUCCESS : EXIT_FAILURE);
+}
-- 
1.6.2.5

