[PATCH] test_ipa_subdom_server: Workaround for slow krb5 + SELinux

[Lukas Slebodnik]

ehlo,

There were failures[1] on rhel6 machine with latest packages.
it took me a while to find out which package cuaed it.
Therefore I downgraded rhel machine to vanilla rhel6.7
and I was troubleshooting it on different machine.

The failures of test_ipa_subdom_server are caused by fixing
memory leak in krb5[2]. BTW there is also plan to fix it in rhel7[3]

sh$ time libtool --mode=execute ./test_ipa_subdom_server

enabled/permissive SELinux
real    0m7.976s
user    0m6.680s
sys     0m0.189s

disabled SELinux
real    0m2.111s
user    0m0.071s
sys     0m0.043s

valgrind + enabled/permissive SELinux //but test failed.
real    2m7.310s
user    2m17.080s
sys     0m0.786s

valgrind + disabled SELinux
real    0m5.510s
user    0m3.396s
sys     0m0.309s

Attached patch "emulates" disabled SELinux.
If we do not want to do that for unit test than we need to increase
few timeouts.
diff --git a/src/providers/ipa/ipa_subdomains_server.c
b/src/providers/ipa/ipa_subdomains_server.c
index f279efc..7d8b3d3 100644
--- a/src/providers/ipa/ipa_subdomains_server.c
+++ b/src/providers/ipa/ipa_subdomains_server.c
@@ -124,7 +124,7 @@ const char *ipa_trust_dir2str(uint32_t direction)
 }

 #ifndef IPA_GETKEYTAB_TIMEOUT
-#define IPA_GETKEYTAB_TIMEOUT 15
+#define IPA_GETKEYTAB_TIMEOUT 5
 #endif /* IPA_GETKEYTAB_TIMEOUT */

 static struct ad_options *
diff --git a/src/tests/cmocka/test_ipa_subdomains_server.c
b/src/tests/cmocka/test_ipa_subdomains_server.c
index d1e0945..3c40f04 100644
--- a/src/tests/cmocka/test_ipa_subdomains_server.c
+++ b/src/tests/cmocka/test_ipa_subdomains_server.c
@@ -508,7 +508,7 @@ static void test_ipa_server_trust_init(void **state)
     ret = ipa_ad_subdom_init(test_ctx->be_ctx, test_ctx->ipa_ctx);
     assert_int_equal(ret, EOK);

-    tv = tevent_timeval_current_ofs(15, 0);
+    tv = tevent_timeval_current_ofs(1, 0);
     timeout_handler = tevent_add_timer(test_ctx->tctx->ev, test_ctx, tv,
                                        ipa_server_init_done, test_ctx);
     assert_non_null(timeout_handler);
@@ -849,7 +849,7 @@ static void test_ipa_server_trust_oneway_init(void **state)
     ret = ipa_ad_subdom_init(test_ctx->be_ctx, test_ctx->ipa_ctx);
     assert_int_equal(ret, EOK);

-    tv = tevent_timeval_current_ofs(15, 0);
+    tv = tevent_timeval_current_ofs(1, 0);
     timeout_handler = tevent_add_timer(test_ctx->tctx->ev, test_ctx, tv,
                                        ipa_server_init_done, test_ctx);
     assert_non_null(timeout_handler);


Lower values caused intermittent failures.

Here are execution times after changing timeouts.

valgrind + enabled/permissive SELinux
real    3m5.812s
user    2m59.929s
sys     0m1.071s

valgrind + disabled SELinux
real    0m33.541s
user    0m3.392s
sys     0m0.322s

disabled SELinux
real    0m30.134s
user    0m0.069s
sys     0m0.040s

enabled/permissive SELinux
real    0m36.014s
user    0m6.768s
sys     0m0.155s

LS

[1] http://sssd-ci.duckdns.org/logs/job/39/10/summary.html
[2] https://bugzilla.redhat.com/show_bug.cgi?id=1311287
[3] https://bugzilla.redhat.com/show_bug.cgi?id=1313457

[Jakub Hrozek]

On Tue, Mar 15, 2016 at 04:47:50PM +0100, Lukas Slebodnik wrote:
>  ehlo,
>  
>  There were failures[1] on rhel6 machine with latest packages.
>  it took me a while to find out which package cuaed it.
>  Therefore I downgraded rhel machine to vanilla rhel6.7
>  and I was troubleshooting it on different machine.
>  
>  The failures of test_ipa_subdom_server are caused by fixing
>  memory leak in krb5[2]. BTW there is also plan to fix it in rhel7[3]
>  
>  sh$ time libtool --mode=execute ./test_ipa_subdom_server
>  
>  enabled/permissive SELinux
>  real    0m7.976s
>  user    0m6.680s
>  sys     0m0.189s
>  
>  disabled SELinux
>  real    0m2.111s
>  user    0m0.071s
>  sys     0m0.043s
>  
>  valgrind + enabled/permissive SELinux //but test failed.
>  real    2m7.310s
>  user    2m17.080s
>  sys     0m0.786s
>  
>  valgrind + disabled SELinux
>  real    0m5.510s
>  user    0m3.396s
>  sys     0m0.309s
>  
>  Attached patch "emulates" disabled SELinux.
>  If we do not want to do that for unit test than we need to increase
>  few timeouts.
>  diff --git a/src/providers/ipa/ipa_subdomains_server.c
> b/src/providers/ipa/ipa_subdomains_server.c
>  index f279efc..7d8b3d3 100644
>  --- a/src/providers/ipa/ipa_subdomains_server.c
>  +++ b/src/providers/ipa/ipa_subdomains_server.c
>  @@ -124,7 +124,7 @@ const char *ipa_trust_dir2str(uint32_t direction)
>   }
>  
>   #ifndef IPA_GETKEYTAB_TIMEOUT
>  -#define IPA_GETKEYTAB_TIMEOUT 15
>  +#define IPA_GETKEYTAB_TIMEOUT 5
>   #endif /* IPA_GETKEYTAB_TIMEOUT */
>  
>   static struct ad_options *
>  diff --git a/src/tests/cmocka/test_ipa_subdomains_server.c
> b/src/tests/cmocka/test_ipa_subdomains_server.c
>  index d1e0945..3c40f04 100644
>  --- a/src/tests/cmocka/test_ipa_subdomains_server.c
>  +++ b/src/tests/cmocka/test_ipa_subdomains_server.c
>  @@ -508,7 +508,7 @@ static void test_ipa_server_trust_init(void **state)
>       ret = ipa_ad_subdom_init(test_ctx->be_ctx, test_ctx->ipa_ctx);
>       assert_int_equal(ret, EOK);
>  
>  -    tv = tevent_timeval_current_ofs(15, 0);
>  +    tv = tevent_timeval_current_ofs(1, 0);
>       timeout_handler = tevent_add_timer(test_ctx->tctx->ev, test_ctx, tv,
>                                          ipa_server_init_done, test_ctx);
>       assert_non_null(timeout_handler);
>  @@ -849,7 +849,7 @@ static void test_ipa_server_trust_oneway_init(void **state)
>       ret = ipa_ad_subdom_init(test_ctx->be_ctx, test_ctx->ipa_ctx);
>       assert_int_equal(ret, EOK);
>  
>  -    tv = tevent_timeval_current_ofs(15, 0);
>  +    tv = tevent_timeval_current_ofs(1, 0);
>       timeout_handler = tevent_add_timer(test_ctx->tctx->ev, test_ctx, tv,
>                                          ipa_server_init_done, test_ctx);
>       assert_non_null(timeout_handler);
>  
>  
>  Lower values caused intermittent failures.
>  
>  Here are execution times after changing timeouts.
>  
>  valgrind + enabled/permissive SELinux
>  real    3m5.812s
>  user    2m59.929s
>  sys     0m1.071s
>  
>  valgrind + disabled SELinux
>  real    0m33.541s
>  user    0m3.392s
>  sys     0m0.322s
>  
>  disabled SELinux
>  real    0m30.134s
>  user    0m0.069s
>  sys     0m0.040s
>  
>  enabled/permissive SELinux
>  real    0m36.014s
>  user    0m6.768s
>  sys     0m0.155s
>  
>  LS
>  
>  [1] http://sssd-ci.duckdns.org/logs/job/39/10/summary.html
>  [2] https://bugzilla.redhat.com/show_bug.cgi?id=1311287
>  [3] https://bugzilla.redhat.com/show_bug.cgi?id=1313457 
>  From 90516358f0a87f7f35dc60e9cb7006f97ff578bd Mon Sep 17 00:00:00
> 2001
>  From: Lukas Slebodnik <lslebodn(a)redhat.com&gt;
>  Date: Tue, 15 Mar 2016 16:14:23 +0100
>  Subject: [PATCH] test_ipa_subdom_server: Workaround for slow krb5 + SELinux
>   
>  diff --git a/src/tests/cmocka/test_ipa_subdomains_server.c
> b/src/tests/cmocka/test_ipa_subdomains_server.c
>  index 3c40f04e3712b22ffc497e08a5b8f38921f4b8b1..0fddc951894dee45658497851473b9bddbba0ef7
> 100644
>  --- a/src/tests/cmocka/test_ipa_subdomains_server.c
>  +++ b/src/tests/cmocka/test_ipa_subdomains_server.c
>  @@ -69,6 +69,26 @@
>   
>   static bool global_rename_called;
>   
>  +#ifdef HAVE_SELINUX
>  +/* Provide faster implementation of kerberos function
>  + * krb5int_labeled_[f]?open. Real functions take care also
>  + * about SELinux context which is very expensive operation
>  + * and cause failures due to timeout when executing with valgrind.
>  + * It's approximately 40 times slower with real function
>  + */
>  +FILE *
>  +krb5int_labeled_fopen(const char *path, const char *mode)
>  +{
>  +    return fopen(path, mode);
>  +}
>  +
>  +int
>  +krb5int_labeled_open(const char *path, int flags, mode_t mode)
>  +{
>  +    return open(path, flags, mode);
>  +}
>  +#endif /* HAVE_SELINUX */
>  + 
This patch seems to work in CI and I don't think we need that specific
functionality in tests, so ACK.

CI: http://sssd-ci.duckdns.org/logs/job/40/80/summary.html

[Lukas Slebodnik]

On (06/04/16 18:44), Jakub Hrozek wrote:
> On Tue, Mar 15, 2016 at 04:47:50PM +0100, Lukas Slebodnik wrote:
> > ehlo,
> > 
> > There were failures[1] on rhel6 machine with latest packages.
> > it took me a while to find out which package cuaed it.
> > Therefore I downgraded rhel machine to vanilla rhel6.7
> > and I was troubleshooting it on different machine.
> > 
> > The failures of test_ipa_subdom_server are caused by fixing
> > memory leak in krb5[2]. BTW there is also plan to fix it in rhel7[3]
> > 
> > sh$ time libtool --mode=execute ./test_ipa_subdom_server
> > 
> > enabled/permissive SELinux
> > real    0m7.976s
> > user    0m6.680s
> > sys     0m0.189s
> > 
> > disabled SELinux
> > real    0m2.111s
> > user    0m0.071s
> > sys     0m0.043s
> > 
> > valgrind + enabled/permissive SELinux //but test failed.
> > real    2m7.310s
> > user    2m17.080s
> > sys     0m0.786s
> > 
> > valgrind + disabled SELinux
> > real    0m5.510s
> > user    0m3.396s
> > sys     0m0.309s
> > 
> > Attached patch "emulates" disabled SELinux.
> > If we do not want to do that for unit test than we need to increase
> > few timeouts.
> > diff --git a/src/providers/ipa/ipa_subdomains_server.c
> b/src/providers/ipa/ipa_subdomains_server.c
> > index f279efc..7d8b3d3 100644
> > --- a/src/providers/ipa/ipa_subdomains_server.c
> > +++ b/src/providers/ipa/ipa_subdomains_server.c
> > @@ -124,7 +124,7 @@ const char *ipa_trust_dir2str(uint32_t direction)
> >  }
> > 
> >  #ifndef IPA_GETKEYTAB_TIMEOUT
> > -#define IPA_GETKEYTAB_TIMEOUT 15
> > +#define IPA_GETKEYTAB_TIMEOUT 5
> >  #endif /* IPA_GETKEYTAB_TIMEOUT */
> > 
> >  static struct ad_options *
> > diff --git a/src/tests/cmocka/test_ipa_subdomains_server.c
> b/src/tests/cmocka/test_ipa_subdomains_server.c
> > index d1e0945..3c40f04 100644
> > --- a/src/tests/cmocka/test_ipa_subdomains_server.c
> > +++ b/src/tests/cmocka/test_ipa_subdomains_server.c
> > @@ -508,7 +508,7 @@ static void test_ipa_server_trust_init(void **state)
> >      ret = ipa_ad_subdom_init(test_ctx->be_ctx, test_ctx->ipa_ctx);
> >      assert_int_equal(ret, EOK);
> > 
> > -    tv = tevent_timeval_current_ofs(15, 0);
> > +    tv = tevent_timeval_current_ofs(1, 0);
> >      timeout_handler = tevent_add_timer(test_ctx->tctx->ev, test_ctx, tv,
> >                                         ipa_server_init_done, test_ctx);
> >      assert_non_null(timeout_handler);
> > @@ -849,7 +849,7 @@ static void test_ipa_server_trust_oneway_init(void **state)
> >      ret = ipa_ad_subdom_init(test_ctx->be_ctx, test_ctx->ipa_ctx);
> >      assert_int_equal(ret, EOK);
> > 
> > -    tv = tevent_timeval_current_ofs(15, 0);
> > +    tv = tevent_timeval_current_ofs(1, 0);
> >      timeout_handler = tevent_add_timer(test_ctx->tctx->ev, test_ctx, tv,
> >                                         ipa_server_init_done, test_ctx);
> >      assert_non_null(timeout_handler);
> > 
> > 
> > Lower values caused intermittent failures.
> > 
> > Here are execution times after changing timeouts.
> > 
> > valgrind + enabled/permissive SELinux
> > real    3m5.812s
> > user    2m59.929s
> > sys     0m1.071s
> > 
> > valgrind + disabled SELinux
> > real    0m33.541s
> > user    0m3.392s
> > sys     0m0.322s
> > 
> > disabled SELinux
> > real    0m30.134s
> > user    0m0.069s
> > sys     0m0.040s
> > 
> > enabled/permissive SELinux
> > real    0m36.014s
> > user    0m6.768s
> > sys     0m0.155s
> > 
> > LS
> > 
> > [1] http://sssd-ci.duckdns.org/logs/job/39/10/summary.html
> > [2] https://bugzilla.redhat.com/show_bug.cgi?id=1311287
> > [3] https://bugzilla.redhat.com/show_bug.cgi?id=1313457
>
> > From 90516358f0a87f7f35dc60e9cb7006f97ff578bd Mon Sep 17 00:00:00 2001
> > From: Lukas Slebodnik <lslebodn(a)redhat.com&gt;
> > Date: Tue, 15 Mar 2016 16:14:23 +0100
> > Subject: [PATCH] test_ipa_subdom_server: Workaround for slow krb5 + SELinux
> > 
>
> > diff --git a/src/tests/cmocka/test_ipa_subdomains_server.c
> b/src/tests/cmocka/test_ipa_subdomains_server.c
> > index
> 3c40f04e3712b22ffc497e08a5b8f38921f4b8b1..0fddc951894dee45658497851473b9bddbba0ef7 100644
> > --- a/src/tests/cmocka/test_ipa_subdomains_server.c
> > +++ b/src/tests/cmocka/test_ipa_subdomains_server.c
> > @@ -69,6 +69,26 @@
> >  
> >  static bool global_rename_called;
> >  
> > +#ifdef HAVE_SELINUX
> > +/* Provide faster implementation of kerberos function
> > + * krb5int_labeled_[f]?open. Real functions take care also
> > + * about SELinux context which is very expensive operation
> > + * and cause failures due to timeout when executing with valgrind.
> > + * It's approximately 40 times slower with real function
> > + */
> > +FILE *
> > +krb5int_labeled_fopen(const char *path, const char *mode)
> > +{
> > +    return fopen(path, mode);
> > +}
> > +
> > +int
> > +krb5int_labeled_open(const char *path, int flags, mode_t mode)
> > +{
> > +    return open(path, flags, mode);
> > +}
> > +#endif /* HAVE_SELINUX */
> > +
>
> This patch seems to work in CI and I don't think we need that specific
> functionality in tests, so ACK.
>
> CI: http://sssd-ci.duckdns.org/logs/job/40/80/summary.html 
Thank you very much for
review.
Now we can upgrade testing machine from rhel6.7 to rhel6.8

LS

[Lukas Slebodnik]

On (07/04/16 09:20), Lukas Slebodnik wrote:
> On (06/04/16 18:44), Jakub Hrozek wrote:
> >On Tue, Mar 15, 2016 at 04:47:50PM +0100, Lukas Slebodnik wrote:
> >> ehlo,
> >> 
> >> There were failures[1] on rhel6 machine with latest packages.
> >> it took me a while to find out which package cuaed it.
> >> Therefore I downgraded rhel machine to vanilla rhel6.7
> >> and I was troubleshooting it on different machine.
> >> 
> >> The failures of test_ipa_subdom_server are caused by fixing
> >> memory leak in krb5[2]. BTW there is also plan to fix it in rhel7[3]
> >> 
> >> sh$ time libtool --mode=execute ./test_ipa_subdom_server
> >> 
> >> enabled/permissive SELinux
> >> real    0m7.976s
> >> user    0m6.680s
> >> sys     0m0.189s
> >> 
> >> disabled SELinux
> >> real    0m2.111s
> >> user    0m0.071s
> >> sys     0m0.043s
> >> 
> >> valgrind + enabled/permissive SELinux //but test failed.
> >> real    2m7.310s
> >> user    2m17.080s
> >> sys     0m0.786s
> >> 
> >> valgrind + disabled SELinux
> >> real    0m5.510s
> >> user    0m3.396s
> >> sys     0m0.309s
> >> 
> >> Attached patch "emulates" disabled SELinux.
> >> If we do not want to do that for unit test than we need to increase
> >> few timeouts.
> >> diff --git a/src/providers/ipa/ipa_subdomains_server.c
> b/src/providers/ipa/ipa_subdomains_server.c
> >> index f279efc..7d8b3d3 100644
> >> --- a/src/providers/ipa/ipa_subdomains_server.c
> >> +++ b/src/providers/ipa/ipa_subdomains_server.c
> >> @@ -124,7 +124,7 @@ const char *ipa_trust_dir2str(uint32_t direction)
> >>  }
> >> 
> >>  #ifndef IPA_GETKEYTAB_TIMEOUT
> >> -#define IPA_GETKEYTAB_TIMEOUT 15
> >> +#define IPA_GETKEYTAB_TIMEOUT 5
> >>  #endif /* IPA_GETKEYTAB_TIMEOUT */
> >> 
> >>  static struct ad_options *
> >> diff --git a/src/tests/cmocka/test_ipa_subdomains_server.c
> b/src/tests/cmocka/test_ipa_subdomains_server.c
> >> index d1e0945..3c40f04 100644
> >> --- a/src/tests/cmocka/test_ipa_subdomains_server.c
> >> +++ b/src/tests/cmocka/test_ipa_subdomains_server.c
> >> @@ -508,7 +508,7 @@ static void test_ipa_server_trust_init(void **state)
> >>      ret = ipa_ad_subdom_init(test_ctx->be_ctx, test_ctx->ipa_ctx);
> >>      assert_int_equal(ret, EOK);
> >> 
> >> -    tv = tevent_timeval_current_ofs(15, 0);
> >> +    tv = tevent_timeval_current_ofs(1, 0);
> >>      timeout_handler = tevent_add_timer(test_ctx->tctx->ev, test_ctx, tv,
> >>                                         ipa_server_init_done, test_ctx);
> >>      assert_non_null(timeout_handler);
> >> @@ -849,7 +849,7 @@ static void test_ipa_server_trust_oneway_init(void **state)
> >>      ret = ipa_ad_subdom_init(test_ctx->be_ctx, test_ctx->ipa_ctx);
> >>      assert_int_equal(ret, EOK);
> >> 
> >> -    tv = tevent_timeval_current_ofs(15, 0);
> >> +    tv = tevent_timeval_current_ofs(1, 0);
> >>      timeout_handler = tevent_add_timer(test_ctx->tctx->ev, test_ctx, tv,
> >>                                         ipa_server_init_done, test_ctx);
> >>      assert_non_null(timeout_handler);
> >> 
> >> 
> >> Lower values caused intermittent failures.
> >> 
> >> Here are execution times after changing timeouts.
> >> 
> >> valgrind + enabled/permissive SELinux
> >> real    3m5.812s
> >> user    2m59.929s
> >> sys     0m1.071s
> >> 
> >> valgrind + disabled SELinux
> >> real    0m33.541s
> >> user    0m3.392s
> >> sys     0m0.322s
> >> 
> >> disabled SELinux
> >> real    0m30.134s
> >> user    0m0.069s
> >> sys     0m0.040s
> >> 
> >> enabled/permissive SELinux
> >> real    0m36.014s
> >> user    0m6.768s
> >> sys     0m0.155s
> >> 
> >> LS
> >> 
> >> [1] http://sssd-ci.duckdns.org/logs/job/39/10/summary.html
> >> [2] https://bugzilla.redhat.com/show_bug.cgi?id=1311287
> >> [3] https://bugzilla.redhat.com/show_bug.cgi?id=1313457
> >
> >> From 90516358f0a87f7f35dc60e9cb7006f97ff578bd Mon Sep 17 00:00:00 2001
> >> From: Lukas Slebodnik <lslebodn(a)redhat.com&gt;
> >> Date: Tue, 15 Mar 2016 16:14:23 +0100
> >> Subject: [PATCH] test_ipa_subdom_server: Workaround for slow krb5 + SELinux
> >> 
> >
> >> diff --git a/src/tests/cmocka/test_ipa_subdomains_server.c
> b/src/tests/cmocka/test_ipa_subdomains_server.c
> >> index
> 3c40f04e3712b22ffc497e08a5b8f38921f4b8b1..0fddc951894dee45658497851473b9bddbba0ef7 100644
> >> --- a/src/tests/cmocka/test_ipa_subdomains_server.c
> >> +++ b/src/tests/cmocka/test_ipa_subdomains_server.c
> >> @@ -69,6 +69,26 @@
> >>  
> >>  static bool global_rename_called;
> >>  
> >> +#ifdef HAVE_SELINUX
> >> +/* Provide faster implementation of kerberos function
> >> + * krb5int_labeled_[f]?open. Real functions take care also
> >> + * about SELinux context which is very expensive operation
> >> + * and cause failures due to timeout when executing with valgrind.
> >> + * It's approximately 40 times slower with real function
> >> + */
> >> +FILE *
> >> +krb5int_labeled_fopen(const char *path, const char *mode)
> >> +{
> >> +    return fopen(path, mode);
> >> +}
> >> +
> >> +int
> >> +krb5int_labeled_open(const char *path, int flags, mode_t mode)
> >> +{
> >> +    return open(path, flags, mode);
> >> +}
> >> +#endif /* HAVE_SELINUX */
> >> +
> >
> >This patch seems to work in CI and I don't think we need that specific
> >functionality in tests, so ACK.
> >
> >CI: http://sssd-ci.duckdns.org/logs/job/40/80/summary.html
> Thank you very much for review.
> Now we can upgrade testing machine from rhel6.7 to rhel6.8
>  
master:
* 1510d1264b44c437b8270e0a5a239e8624933c3d

sssd-1-13:
* 67a485deee450a4bc7c48af23a8f851f2e7960f8

LS