[389-ds-base] branch master updated (c0346d5 -> ab61eff)

[git repository hosting]

This is an automated email from the git hooks/post-receive script.

firstyear pushed a change to branch master
in repository 389-ds-base.

from  c0346d5   Ticket 49495 - Fix memory management is vattr.
       new  ab61eff   Ticket 49495 - cos stress test and improvements.

The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails.  The revisions
listed as "adds" were already present in the repository and have only
been added to this reference.

Summary of changes:
 .../tests/stress/cos/cos_scale_template_test.py    | 148 +++++++++++++++++++++
 src/lib389/lib389/cos.py                           | 146 ++++++++++++++++++++
 2 files changed, 294 insertions(+)
 create mode 100644 dirsrvtests/tests/stress/cos/cos_scale_template_test.py
 create mode 100644 src/lib389/lib389/cos.py

[389-ds-base] 01/01: Ticket 49495 - cos stress test and improvements.

[git repository hosting]

This is an automated email from the git hooks/post-receive script.

firstyear pushed a commit to branch master
in repository 389-ds-base.

commit ab61eff1dfae54e84419fd4c059a3bae902fdf95
Author: William Brown 
Date:   Mon Dec 11 13:16:33 2017 +0100

Ticket 49495 - cos stress test and improvements.

Bug Description:  We previously had no way to test the cos plugin.

Fix Description:  Add cos types, and a stress test for the
    template system to demonstrate the issue with 49495

https://pagure.io/389-ds-base/issue/49495

Author: wibrown

Review by: spichugi (Thanks!)
---
 .../tests/stress/cos/cos_scale_template_test.py    | 148 +++++++++++++++++++++
 src/lib389/lib389/cos.py                           | 146 ++++++++++++++++++++
 2 files changed, 294 insertions(+)

diff --git a/dirsrvtests/tests/stress/cos/cos_scale_template_test.py b/dirsrvtests/tests/stress/cos/cos_scale_template_test.py
new file mode 100644
index 0000000..c01aac5
--- /dev/null
+++ b/dirsrvtests/tests/stress/cos/cos_scale_template_test.py
@@ -0,0 +1,148 @@
+# --- BEGIN COPYRIGHT BLOCK ---
+# Copyright (C) 2017 Red Hat, Inc.
+# All rights reserved.
+#
+# License: GPL (version 3 or any later version).
+# See LICENSE for details.
+# --- END COPYRIGHT BLOCK ---
+#
+
+import pytest
+
+from lib389.topologies import topology_st
+
+from lib389.plugins import ClassOfServicePlugin
+from lib389.cos import CosIndirectDefinitions, CosTemplates, CosTemplate
+from lib389.idm.user import UserAccounts, TEST_USER_PROPERTIES
+from lib389.idm.organisationalunit import OrganisationalUnits
+
+from lib389._constants import DEFAULT_SUFFIX
+
+import time
+
+# Given this should complete is about 0.005, this is generous.
+# For the final test with 20 templates, about 0.02 is an acceptable time.
+THRESHOLD = 0.05
+
+class OUCosTemplate(CosTemplate):
+    def __init__(self, instance, dn=None):
+        """Create a OU specific cos template to replicate a specific user setup.
+        This template provides ou attrs onto the target entry.
+
+        :param instance: A dirsrv instance
+        :type instance: DirSrv
+        :param dn: The dn of the template
+        :type dn: str
+        """
+        super(OUCosTemplate, self).__init__(instance, dn)
+        self._rdn_attribute = 'ou'
+        self._must_attributes = ['ou']
+        self._create_objectclasses = [
+            'top',
+            'cosTemplate',
+            'organizationalUnit',
+        ]
+
+class OUCosTemplates(CosTemplates):
+    def __init__(self, instance, basedn, rdn=None):
+        """Create an OU specific cos templates to replicate a specific use setup.
+        This costemplates object allows access to the OUCosTemplate types.
+
+        :param instance: A dirsrv instance
+        :type instance: DirSrv
+        :param basedn: The basedn of the templates
+        :type basedn: str
+        :param rdn: The rdn of the templates
+        :type rdn: str
+        """
+        super(OUCosTemplates, self).__init__(instance, basedn, rdn)
+        self._objectclasses = [
+            'cosTemplate',
+            'organizationalUnit',
+        ]
+        self._filterattrs = ['ou']
+        self._childobject = OUCosTemplate
+
+def test_indirect_template_scale(topology_st):
+    """Test that cos templates can be added at a reasonable scale
+
+    :id: 7cbcdf22-1f9c-4222-9e76-685fe374fc20
+    :steps:
+        1. Enable COS plugin
+        2. Create the test user
+        3. Add an indirect cos template
+        4. Add a cos template
+        5. Add the user to the cos template and assert it works.
+        6. Add 25,000 templates to the database
+        7. Search the user. It should not exceed THRESHOLD.
+    :expected results:
+        1. It is enabled.
+        2. It is created.
+        3. Is is created.
+        4. It is created.
+        5. It is valid.
+        6. They are created.
+        7. It is fast.
+    """
+
+    cos_plugin = ClassOfServicePlugin(topology_st.standalone)
+    cos_plugin.enable()
+
+    topology_st.standalone.restart()
+
+    # Now create, the indirect specifier, and a user to template onto.
+    users = UserAccounts(topology_st.standalone, DEFAULT_SUFFIX)
+    user = users.create(properties=TEST_USER_PROPERTIES)
+
+    cos_inds = CosIndirectDefinitions(topology_st.standalone, DEFAULT_SUFFIX)
+    cos_ind = cos_inds.create(properties={
+        'cn' : 'cosIndirectDef',
+        'cosIndirectSpecifier': 'seeAlso',
+        'cosAttribute': [
+            'ou merge-schemes',
+            'description merge-schemes',
+            'postalCode merge-schemes',
+            ],
+    })
+
+    ous = OrganisationalUnits(topology_st.standalone, DEFAULT_SUFFIX)
+    ou_temp = ous.create(properties={'ou': 'templates'})
+    cos_temps = OUCosTemplates(topology_st.standalone, ou_temp.dn)
+
+    cos_temp_u = cos_temps.create(properties={
+        'ou' : 'ou_temp_u',
+        'description' : 'desc_temp_u',
+        'postalCode': '0'
+    })
+    # Edit the user to add the seeAlso ...
+    user.set('seeAlso', cos_temp_u.dn)
+
+    # Now create 25,0000 templates, they *don't* need to apply to the user though!
+    for i in range(1, 25001):
+        cos_temp_u = cos_temps.create(properties={
+            'ou' : 'ou_temp_%s' % i,
+            'description' : 'desc_temp_%s' % i,
+            'postalCode': '%s' % i
+        })
+
+        if i % 500 == 0:
+            start_time = time.monotonic()
+            u_search = users.get('testuser')
+            attrs = u_search.get_attr_vals_utf8('postalCode')
+            end_time = time.monotonic()
+            diff_time = end_time - start_time
+            assert diff_time < THRESHOLD
+
+        if i == 10000:
+            # Now add our user to this template also.
+            user.add('seeAlso', cos_temp_u.dn)
+
+            start_time = time.monotonic()
+            attrs_after = u_search.get_attr_vals_utf8('postalCode')
+            end_time = time.monotonic()
+            diff_time = end_time - start_time
+            assert(set(attrs) < set(attrs_after))
+            assert diff_time < THRESHOLD
+
+
+
diff --git a/src/lib389/lib389/cos.py b/src/lib389/lib389/cos.py
new file mode 100644
index 0000000..783e5e5
--- /dev/null
+++ b/src/lib389/lib389/cos.py
@@ -0,0 +1,146 @@
+# --- BEGIN COPYRIGHT BLOCK ---
+# Copyright (C) 2017 Red Hat, Inc.
+# All rights reserved.
+#
+# License: GPL (version 3 or any later version).
+# See LICENSE for details.
+# --- END COPYRIGHT BLOCK ---
+#
+
+# Implement types for COS handling with lib389 and DS.
+
+
+from lib389._mapped_object import DSLdapObject, DSLdapObjects
+
+from lib389.utils import ensure_str
+
+class CosTemplate(DSLdapObject):
+    def __init__(self, instance, dn=None):
+        """A Cos Template defining the values to override on a target.
+
+        :param instance: DirSrv instance
+        :type instance: DirSrv
+        :param dn: The dn of the template
+        :type dn: str
+        """
+        super(CosTemplate, self).__init__(instance, dn)
+        self._rdn_attribute = 'cn'
+        self._must_attributes = ['cn']
+        # This is the ONLY TIME i'll allow extensible object ...
+        # You have been warned ...
+        self._create_objectclasses = [
+            'top',
+            'cosTemplate',
+            'extensibleObject',
+        ]
+        self._protected = False
+
+class CosTemplates(DSLdapObjects):
+    def __init__(self, instance, basedn, rdn=None):
+        """The set of costemplates that exist for direct and indirect
+        implementations.
+
+        :param instance: A dirsrv instance
+        :type instance: DirSrv
+        :param basedn: The basedn of the templates
+        :type basedn: str
+        :param rdn: The rdn of the templates
+        :type rdn: str
+        """
+        super(CosTemplates, self).__init__(instance)
+        self._objectclasses = [
+            'cosTemplate'
+        ]
+        self._filterattrs = ['cn']
+        self._childobject = CosTemplate
+        self._basedn = basedn
+        if rdn is not None:
+            self._basedn = '{},{}'.format(ensure_str(rdn), ensure_str(basedn))
+
+
+class CosIndirectDefinition(DSLdapObject):
+    def __init__(self, instance, dn=None):
+        """A Cos Indirect Definition associating an attr:value pair as a link
+        attr to a template type.
+
+        :param instance: DirSrv instance
+        :type instance: DirSrv
+        :param dn: The dn of the template
+        :type dn: str
+        """
+        super(CosIndirectDefinition, self).__init__(instance, dn)
+        self._rdn_attribute = 'cn'
+        self._must_attributes = ['cn', 'cosIndirectSpecifier', 'cosAttribute']
+        self._create_objectclasses = [
+            'top',
+            'cosSuperDefinition',
+            'cosIndirectDefinition',
+        ]
+        self._protected = False
+
+class CosIndirectDefinitions(DSLdapObjects):
+    def __init__(self, instance, basedn, rdn=None):
+        """The set of cos indirect definitions that exist.
+
+        :param instance: A dirsrv instance
+        :type instance: DirSrv
+        :param basedn: The basedn of the templates
+        :type basedn: str
+        :param rdn: The rdn of the templates
+        :type rdn: str
+        """
+        super(CosIndirectDefinitions, self).__init__(instance)
+        self._objectclasses = [
+            'cosSuperDefinition',
+            'cosIndirectDefinition',
+        ]
+        self._filterattrs = ['cn']
+        self._childobject = CosIndirectDefinition
+        self._basedn = basedn
+        if rdn is not None:
+            self._basedn = '{},{}'.format(ensure_str(rdn), ensure_str(basedn))
+
+
+class CosPointerDefinition(DSLdapObject):
+    def __init__(self, instance, dn=None):
+        """A Cos Pointer Definition associating a dn syntax type as a link
+        attr to a template type.
+
+        :param instance: DirSrv instance
+        :type instance: DirSrv
+        :param dn: The dn of the template
+        :type dn: str
+        """
+        super(CosPointerDefinition, self).__init__(instance, dn)
+        self._rdn_attribute = 'cn'
+        self._must_attributes = ['cn', 'cosTemplateDn', 'cosAttribute']
+        self._create_objectclasses = [
+            'top',
+            'cosSuperDefinition',
+            'cosPointerDefinition',
+        ]
+        self._protected = False
+
+class CosPointerDefinitions(DSLdapObjects):
+    def __init__(self, instance, basedn, rdn=None):
+        """The set of cos pointer definitions that exist.
+
+        :param instance: A dirsrv instance
+        :type instance: DirSrv
+        :param basedn: The basedn of the templates
+        :type basedn: str
+        :param rdn: The rdn of the templates
+        :type rdn: str
+        """
+        super(CosPointerDefinitions, self).__init__(instance)
+        self._objectclasses = [
+            'cosSuperDefinition',
+            'cosPointerDefinition',
+        ]
+        self._filterattrs = ['cn']
+        self._childobject = CosPointerDefinition
+        self._basedn = basedn
+        if rdn is not None:
+            self._basedn = '{},{}'.format(ensure_str(rdn), ensure_str(basedn))
+
+