Add dracut-memdebug-ko.sh, install it to the dracut kdump module.
The principle is to use kernel trace to track slab and buddy allocation
calls during kernel module loading(module_init), thus we can analyze
all the trace data and get the total memory consumption.
One major flaw of this method is that it consumes a lot of memory,
users should increase the crash kernel memory reservation as needed.
Signed-off-by: Xunlei Pang <xlpang(a)redhat.com>
---
dracut-memdebug-ko.sh | 151 ++++++++++++++++++++++++++++++++++++++++++++++++++
kexec-tools.spec | 2 +
2 files changed, 153 insertions(+)
create mode 100755 dracut-memdebug-ko.sh
diff --git a/dracut-memdebug-ko.sh b/dracut-memdebug-ko.sh
new file mode 100755
index 0000000..bb404ce
--- /dev/null
+++ b/dracut-memdebug-ko.sh
@@ -0,0 +1,151 @@
+#Debug the number of memory through alloc_pages and kmalloc consumed
+#by kernel modules during loading(i.e. modprobe).
+#NOTE: kmalloc may trigger alloc_pages, thus resulting in double account.
+
+if ! [[ -e /sys/kernel/debug/tracing ]]; then
+ mount none -t debugfs /sys/kernel/debug
+ if ! [[ -d /sys/kernel/debug/tracing ]]; then
+ warn "Mount debugfs failed, can't activate trace, skip kernel module memory
analyzing!"
+ return 0
+ fi
+
+ # 10MB should be big enough for most cases?
+ # If the current ring buffer size is the default one(contains "expanded"
keyword),
+ # set it to 10MB. Users can set other size via "trace_buf_size" kernel boot
command.
+ cat /proc/cmdline | grep -q "trace_buf_size="
+ if [[ $? -ne 0 ]]; then
+ echo 10240 > /sys/kernel/debug/tracing/buffer_size_kb
+ fi
+
+ # Prepare trace for the first time
+ echo 1 > /sys/kernel/debug/tracing/events/kmem/mm_page_alloc/enable
+ echo 1 > /sys/kernel/debug/tracing/events/kmem/mm_page_free/enable
+ echo 1 > /sys/kernel/debug/tracing/events/kmem/kmalloc/enable
+ echo 1 > /sys/kernel/debug/tracing/events/kmem/kmalloc_node/enable
+ echo 1 > /sys/kernel/debug/tracing/events/kmem/kmem_cache_alloc/enable
+ echo 1 > /sys/kernel/debug/tracing/events/kmem/kmem_cache_alloc_node/enable
+ echo 1 > /sys/kernel/debug/tracing/events/module/module_load/enable
+ echo 1 > /sys/kernel/debug/tracing/events/module/module_put/enable
+ echo 1 > /sys/kernel/debug/tracing/tracing_on
+ return 0
+fi
+
+echo 0 > /sys/kernel/debug/tracing/tracing_on
+TMPFILE=/tmp/tmp$$$$
+cp /sys/kernel/debug/tracing/trace $TMPFILE -f
+# Clear old trace data after copied away
+echo > /sys/kernel/debug/tracing/trace
+
+#Indexed by task pid.
+declare -A current_module
+
+#Indexed by module name.
+declare -A module_loaded
+declare -A nr_alloc_pages
+declare -A nr_alloc_pages_peak
+declare -A nr_kmalloc
+#For x86: If the request size of kmalloc is greater than 2*PAGE_SIZE, SLUB will use buddy
instead.
+#So we maintain the statistics of the large kmalloc requests. For ppc64, PAGE_SIZE is not
4096,
+#but the large kmalloc request is not very common, this information is just to give some
tips.
+declare -A nr_kmalloc_above8192
+
+declare -A nr_kmem_cache_alloc
+
+# $1: order of pages
+order_to_pages()
+{
+ local pages=1
+ local order=$1
+
+ while [[ $order != 0 ]]; do
+ order=$((order-1))
+ pages=$(($pages*2))
+ done
+
+ echo $pages
+}
+
+while read pid cpu flags ts function ;
+do
+ #Skip comment lines
+ if [[ $pid = "#" ]]; then
+ continue
+ fi
+
+ if [[ $function = module_load* ]]; then
+ #One module is being loaded, save the task pid for tracking.
+ module_name=${function#*: }
+ module_names+=" $module_name"
+ current_module[$pid]="$module_name"
+ [[ ${module_loaded[$module_name]} ]] && warn "\"$module_name\"
was loaded multiple times!"
+ unset module_loaded[$module_name]
+ nr_alloc_pages[$module_name]=0
+ nr_alloc_pages_peak[$module_name]=0
+ nr_kmalloc[$module_name]=0
+ nr_kmalloc_above8192[$module_name]=0
+ nr_kmem_cache_alloc[$module_name]=0
+ fi
+
+ if ! [[ ${current_module[$pid]} ]]; then
+ continue
+ fi
+
+ if [[ $function = module_put* ]]; then
+ #Mark the module as loaded
+ module_loaded[${current_module[$pid]}]=1
+ #module has been loaded when module_put is called, untrack the task
+ unset current_module[$pid]
+ continue
+ fi
+
+ #Once we get here, the task is being tracked(is loading a module).
+ #Get the module name.
+ module_name=${current_module[$pid]}
+
+ if [[ $function = mm_page_alloc* ]]; then
+ order=$(echo $function | sed -e 's/.*order=\([0-9]*\) .*/\1/')
+ nr_alloc_pages[$module_name]=$((${nr_alloc_pages[$module_name]}+$(order_to_pages
$order)))
+ if [[ ${nr_alloc_pages[$module_name]} -gt ${nr_alloc_pages_peak[$module_name]} ]];
then
+ nr_alloc_pages_peak[$module_name]=${nr_alloc_pages[$module_name]}
+ fi
+ fi
+
+ if [[ $function = mm_page_free* ]]; then
+ order=$(echo $function | sed -e 's/.*order=\([0-9]*\)/\1/')
+ nr_alloc_pages[$module_name]=$((${nr_alloc_pages[$module_name]}-$(order_to_pages
$order)))
+ fi
+
+ if [[ $function = kmalloc* ]]; then
+ bytes_alloc=$(echo $function | sed -e 's/.*bytes_alloc=\([0-9]*\).*/\1/')
+ nr_kmalloc[$module_name]=$((${nr_kmalloc[$module_name]}+$bytes_alloc))
+ if [[ $bytes_alloc -gt 8192 ]]; then
+ nr_kmalloc_above8192[$module_name]=$((${nr_kmalloc_above8192[$module_name]}+$bytes_alloc))
+ fi
+ fi
+
+ if [[ $function = kmem_cache_alloc* ]]; then
+ bytes_alloc=$(echo $function | sed -e 's/.*bytes_alloc=\([0-9]*\).*/\1/')
+ nr_kmem_cache_alloc[$module_name]=$((${nr_kmem_cache_alloc[$module_name]}+$bytes_alloc))
+ fi
+done < $TMPFILE
+
+echo -e "\n\n== debug_mem for kernel modules during loading begin =="
>&2
+for i in $module_names; do
+ status="finished"
+ if ! [[ ${module_loaded[$i]} ]]; then
+ status="loading"
+ fi
+ echo -e "[Module Name] \"$i\" (loading status: $status)" >&2
+ echo -e "alloc_pages: consumed ${nr_alloc_pages[$i]} pages (peak:
${nr_alloc_pages_peak[$i]} pages)" >&2
+ echo -e "kmalloc: consumed ${nr_kmalloc[$i]} bytes (above8192:
${nr_kmalloc_above8192[$i]} bytes)\n" >&2
+ echo -e "kmem_cache_alloc: consumed ${nr_kmem_cache_alloc[$i]} bytes\n"
>&2
+done
+echo -e "== debug_mem for kernel modules during loading end ==\n\n" >&2
+
+unset module_names
+unset module_loaded
+
+rm $TMPFILE -f
+echo 1 > /sys/kernel/debug/tracing/tracing_on
+
+return 0
diff --git a/kexec-tools.spec b/kexec-tools.spec
index 0bbaf72..1f0b7f5 100644
--- a/kexec-tools.spec
+++ b/kexec-tools.spec
@@ -40,6 +40,7 @@ Source103: dracut-kdump-error-handler.sh
Source104: dracut-kdump-emergency.service
Source105: dracut-kdump-error-handler.service
Source106: dracut-kdump-capture.service
+Source107: dracut-memdebug-ko.sh
Requires(post): systemd-units
Requires(preun): systemd-units
@@ -203,6 +204,7 @@ cp %{SOURCE103}
$RPM_BUILD_ROOT/etc/kdump-adv-conf/kdump_dracut_modules/99kdumpb
cp %{SOURCE104}
$RPM_BUILD_ROOT/etc/kdump-adv-conf/kdump_dracut_modules/99kdumpbase/%{remove_dracut_prefix
%{SOURCE104}}
cp %{SOURCE105}
$RPM_BUILD_ROOT/etc/kdump-adv-conf/kdump_dracut_modules/99kdumpbase/%{remove_dracut_prefix
%{SOURCE105}}
cp %{SOURCE106}
$RPM_BUILD_ROOT/etc/kdump-adv-conf/kdump_dracut_modules/99kdumpbase/%{remove_dracut_prefix
%{SOURCE106}}
+cp %{SOURCE107}
$RPM_BUILD_ROOT/etc/kdump-adv-conf/kdump_dracut_modules/99kdumpbase/%{remove_dracut_prefix
%{SOURCE107}}
chmod 755
$RPM_BUILD_ROOT/etc/kdump-adv-conf/kdump_dracut_modules/99kdumpbase/%{remove_dracut_prefix
%{SOURCE100}}
chmod 755
$RPM_BUILD_ROOT/etc/kdump-adv-conf/kdump_dracut_modules/99kdumpbase/%{remove_dracut_prefix
%{SOURCE101}}
--
1.8.3.1