Change in vdsm[master]: vdsm: implement NUMA aware KSM policy - vdsm-patches - Fedora mailing-lists

22 Apr 2015

Dudi Maroshi has uploaded a new change for review.
Change subject: vdsm: implement NUMA aware KSM policy
......................................................................
vdsm: implement NUMA aware KSM policy
Add monitored parameter ksm_merge_across_nodes to:
1. momIf.py class - append to status report
2. superVdsmServer file - enable writing to the kernel flag
Update policy logic (a simple state machine) to 03-ksm.policy
merge_across_nodes flag allowed to be changed only after unmerging
all pages. This unmerge completion is tested on pages_shared flag == 0.
Update policy parameter ksmMergeAcrossNodes to 00-defines.policy
Change-Id: Iec607f9ef3284c1448bfc2831d125fc4d81b28d2
Related-to: https://bugzilla.redhat.com/show_bug.cgi?id=840114
Signed-off-by: Dudi Maroshi dudi@redhat.com
---
M vdsm/mom.d/00-defines.policy
M vdsm/mom.d/03-ksm.policy
M vdsm/momIF.py
M vdsm/supervdsmServer
4 files changed, 73 insertions(+), 53 deletions(-)
git pull ssh://gerrit.ovirt.org:29418/vdsm refs/changes/29/40129/1

diff --git a/vdsm/mom.d/00-defines.policy b/vdsm/mom.d/00-defines.policy
index faa6d90..718cf04 100644
--- a/vdsm/mom.d/00-defines.policy
+++ b/vdsm/mom.d/00-defines.policy
@@ -4,6 +4,7 @@
 (defvar True 1)
# Define variables for configurable options here
+(defvar ksmMergeAcrossNodes 1)
 (defvar ksmEnabled 1)
 (defvar balloonEnabled 0)
 (defvar cpuTuneEnabled 1)
diff --git a/vdsm/mom.d/03-ksm.policy b/vdsm/mom.d/03-ksm.policy
index 5a3b1bd..9706e5d 100644
--- a/vdsm/mom.d/03-ksm.policy
+++ b/vdsm/mom.d/03-ksm.policy
@@ -22,6 +22,10 @@
 # KSM will be started to try and free up some memory.
 (defvar ksm_free_percent 0.20)
+# KSM service can unmerged shared pages (ksm_run = 2)
+# When this happens the KSM is affectivley non responsive
+(defvar ksm_umerging_pages 0)
+
 ### Helper functions
 (def change_npages (delta)
 {
@@ -31,6 +35,43 @@
     (Host.Control "ksm_pages_to_scan" newval)
 })
+(def enable_ksmMergeAcrossNodes()
+{
+    (debug "entry: enable_ksmMergeAcrossNodes")
+    (debug Host.ksm_run "=ksm_run")
+    (debug Host.ksm_merge_across_nodes "=ksm_merge_across_nodes")
+    (debug ksmMergeAcrossNodes "=ksmMergeAcrossNodes")
+    (if (!= Host.ksm_merge_across_nodes ksmMergeAcrossNodes){
+        (debug "ksm_merge_across_nodes policy change")                                                             
+        (if (== Host.ksm_run 0){
+            (debug "ksm_run == 0 : Update ksm_merge_across_nodes")
+            (Host.Control "ksm_merge_across_nodes" ksmMergeAcrossNodes)
+            (debug "ksm_merge_across_nodes configured")}
+          6)
+        (if (== Host.ksm_run 1){
+            (debug "ksm_run == 1  : unmerge KSM pages")
+            (setq ksm_umerging_pages 1)
+            (Host.Control "ksm_run" 2)
+            (debug "ksm_run unmerge")}
+          7)
+        (if (== Host.ksm_run 2){
+            (debug "ksm_run == 2 : if all pages unmerged, update ksm_merge_across_nodes. Rest KSM")
+            (Host.Control "ksm_run" 2)
+            (if (== Host.ksm_pages_shared 0) {
+                    (debug "ksm_pages_shared == 0")
+                    (Host.Control "ksm_merge_across_nodes" ksmMergeAcrossNodes)
+                    (Host.Control "ksm_run" 0)
+                    (setq ksm_umerging_pages 0)
+                    } 
+                    { #else
+                    (debug "ksm_pages_shared != 0 : continue with unmerging")
+                    (setq ksm_umerging_pages 1)
+                    })
+            (debug "ksm_merge_across_nodes configured")}
+          9)
+     } 10)
+    (debug "exit: enable_ksmMergeAcrossNodes")
+})
 ### Main Script
 # Methodology: Since running KSM does incur some overhead, try to run it only
 # when necessary.  If the amount of committed KSM shareable memory is high or if
@@ -40,16 +81,19 @@
(defvar ksm_pressure_threshold (* Host.mem_available ksm_free_percent))
 (defvar ksm_committed Host.ksm_shareable)
-
-(if (or (and (< (+ ksm_pressure_threshold ksm_committed) Host.mem_available)
-         (> (Host.StatAvg "mem_free") ksm_pressure_threshold)) (not ksmEnabled))
-    (Host.Control "ksm_run" 0)
-    {        # else
-        (Host.Control "ksm_run" 1)
-        (Host.Control "ksm_sleep_millisecs"
-            (/ (* ksm_sleep_ms_baseline 16777216) Host.mem_available))
-       (if (< (Host.StatAvg "mem_free") ksm_pressure_threshold)
-            (change_npages ksm_pages_boost)
-           (change_npages ksm_pages_decay))
-    }
-)
+(enable_ksmMergeAcrossNodes)
+(if (== ksm_umerging_pages 0)
+    (if (or (and (< (+ ksm_pressure_threshold ksm_committed) Host.mem_available)
+                 (> (Host.StatAvg "mem_free") ksm_pressure_threshold)) 
+             (not ksmEnabled))
+        (Host.Control "ksm_run" 0)
+        {        # else
+            (Host.Control "ksm_run" 1)
+            (Host.Control "ksm_sleep_millisecs"
+                (/ (* ksm_sleep_ms_baseline 16777216) Host.mem_available))
+           (if (< (Host.StatAvg "mem_free") ksm_pressure_threshold)
+               (change_npages ksm_pages_boost)
+               (change_npages ksm_pages_decay))
+        }
+    )
+0)
\ No newline at end of file
diff --git a/vdsm/momIF.py b/vdsm/momIF.py
index d70d826..e6481c2 100644
--- a/vdsm/momIF.py
+++ b/vdsm/momIF.py
@@ -56,6 +56,7 @@
         stats = self._mom.getStatistics()['host']
         ret = {}
         ret['ksmState'] = bool(stats['ksm_run'])
+        ret['ksmMergeAcrossNodes'] = bool(stats['ksm_merge_across_nodes'])
         ret['ksmPages'] = stats['ksm_pages_to_scan']
         ret['memShared'] = stats['ksm_pages_sharing'] * PAGE_SIZE_BYTES
         ret['memShared'] /= Mbytes
diff --git a/vdsm/supervdsmServer b/vdsm/supervdsmServer
index 3cdbf71..43ed1e6 100755
--- a/vdsm/supervdsmServer
+++ b/vdsm/supervdsmServer
@@ -308,25 +308,14 @@
         of /dev/vfio/$iommu_group to qemu:qemu. This method should be called
         when detaching a device from the host.
         """
-        rule_file = _UDEV_RULE_FILE_NAME_VFIO % iommu_group
-
-        if not os.path.isfile(rule_file):
-            # If the file exists, different device from the same group has
-            # already been detached and we therefore can skip overwriting the
-            # file. Also, this file should only be created/removed via the
-            # means of supervdsm.
-
-            rule = ('KERNEL=="{}", SUBSYSTEM=="vfio" RUN+="{} {}:{} '
-                    '/dev/vfio/{}"').format(iommu_group, EXT_CHOWN,
-                                            QEMU_PROCESS_USER,
-                                            QEMU_PROCESS_GROUP,
-                                            iommu_group)
-
-            with open(rule_file, "w") as rf:
-                self.log.debug("Creating rule %s: %r", rule_file, rule)
-                rf.write(rule)
-
-            self.udevTrigger(iommu_group)
+        ruleFile = _UDEV_RULE_FILE_NAME_VFIO % iommu_group
+        rule = ('KERNEL=="{}", SUBSYSTEM=="vfio" RUN+="{} {}:{} '
+                '/dev/vfio/{}"').format(iommu_group, EXT_CHOWN,
+                                        QEMU_PROCESS_USER, QEMU_PROCESS_GROUP,
+                                        iommu_group)
+        with open(ruleFile, "w") as rf:
+            self.log.debug("Creating rule %s: %r", ruleFile, rule)
+            rf.write(rule)
@logDecorator
     def rmAppropriateIommuGroup(self, iommu_group):
@@ -334,26 +323,10 @@
         Remove udev rule in /etc/udev/rules.d/ created by
         vfioAppropriateDevice.
         """
-        rule_file = os.path.join(_UDEV_RULE_FILE_DIR, _UDEV_RULE_FILE_PREFIX +
-                                 "iommu_group_" + iommu_group +
-                                 _UDEV_RULE_FILE_EXT)
-        error = False
-
-        try:
-            os.remove(rule_file)
-        except OSError as e:
-            if e.errno == errno.ENOENT:
-                # OSError with ENOENT errno here means that the rule file does
-                # not exist - this is expected when multiple devices in one
-                # iommu group were passed through.
-                error = True
-            else:
-                raise
-        else:
-            self.log.debug("Removing rule %s", rule_file)
-
-        if not error:
-            self.udevTrigger(iommu_group)
+        rule = os.path.join(_UDEV_RULE_FILE_DIR, _UDEV_RULE_FILE_PREFIX +
+                            "iommu_group_" + iommu_group + _UDEV_RULE_FILE_EXT)
+        self.log.debug("Removing rule %s", rule)
+        os.remove(rule)
@logDecorator
     def ksmTune(self, tuningParams):
@@ -362,7 +335,8 @@
         when it's lauched by vdsm. So it needs supervdsm's assistance to tune
         KSM's parameters.
         '''
-        KSM_PARAMS = {'run': 3, 'sleep_millisecs': 0x100000000,
+        KSM_PARAMS = {'run': 3, 'merge_across_nodes': 3,
+                      'sleep_millisecs': 0x100000000,
                       'pages_to_scan': 0x100000000}
         for (k, v) in tuningParams.iteritems():
             if k not in KSM_PARAMS.iterkeys():
-- 
To view, visit https://gerrit.ovirt.org/40129
To unsubscribe, visit https://gerrit.ovirt.org/settings

Gerrit-MessageType: newchange
Gerrit-Change-Id: Iec607f9ef3284c1448bfc2831d125fc4d81b28d2
Gerrit-PatchSet: 1
Gerrit-Project: vdsm
Gerrit-Branch: master
Gerrit-Owner: Dudi Maroshi dudi@redhat.com

    