RE: [fedora-virt] f10 x86_64 xen VM guests fail to boot on f8 host(guest setting NX bit in L1 PTE?)

[Jon Swanson]

Hey Ian,

Probably incompetence on my part, but I am unable to get the patch to
apply.  What specific version of the kernel are you doing this on?

I'm getting the following error:
------------------------------------------------------------------------
---------------------------------------------
~/rpmbuild/BUILD> patch --dry-run -p0 < ~/f10xenNoNX.patch.txt
patching file a/arch/x86/mm/init_64.c
patch: **** malformed patch at line 19: @@ -121,8 +124,13 @@
------------------------------------------------------------------------
---------------------------------------------



I've tried with the following kernel versions, and duplicated your
directory structure:
kernel-2.6.27.5-117.fc10.src.rpm  
kernel-2.6.27.9-159.fc10.src.rpm

I've also tried just patching one of the files:
------------------------------------------------------------------------
---------------------------------------------
~/rpmbuild/BUILD>patch --dry-run --verbose a/arch/x86/mm/init_64.c
~/f10init_64.c.patch
Hmm...  Looks like a unified diff to me...
The text leading up to this was:
--------------------------
|diff -r ec792b22009f arch/x86/mm/init_64.c
|--- a/arch/x86/mm/init_64.c    Fri Jan 23 15:27:45 2009 +0000
|+++ b/arch/x86/mm/init_64.c    Fri Jan 23 15:58:03 2009 +0000
--------------------------
Patching file a/arch/x86/mm/init_64.c using Plan A...
patch: **** malformed patch at line 19: @@ -121,8 +124,13 @@
------------------------------------------------------------------------
---------------------------------------------




I opened up the file and was about to do it manually, but it seems
radically different so stopped.
------------------------------------------------------------------------
---------------------------------------------
 ~/rpmbuild/BUILD> sed -n '120,125p' a/arch/x86/mm/init_64.c

        pud = pud_page + pud_index(vaddr);
        if (pud_none(*pud)) {
                pmd = (pmd_t *) spp_getpage();
                pud_populate(&init_mm, pud, pmd);
                if (pmd != pmd_offset(pud, 0)) {
------------------------------------------------------------------------
---------------------------------------------


I'm sorry if I'm just being an idiot, but any insight you can share
would be greatly appreciated.

Thanks,
 jon 

-----Original Message-----
From: Ian Campbell [mailto:[e-mail]] 
Sent: Saturday, January 24, 2009 12:59 AM
To: Jon Swanson
Cc: Mark McLoughlin; fedora-virt(a)redhat.com;
virtualization(a)webwombat.com.au; Jeremy Fitzhardinge
Subject: RE: [fedora-virt] f10 x86_64 xen VM guests fail to boot on f8
host(guest setting NX bit in L1 PTE?)

On Wed, 2009-01-21 at 12:22 +0900, Jon Swanson wrote:
>  Thank you again Mark, Ian, and Phil.
>  
>  As Phil pointed out, my working host has the NX feature, the broken 
>  host does not.
>  
>  I also could not create a bugzilla account, but was able to use a 
>  co-worker's credentials to create this ticket:
>  https://bugzilla.redhat.com/show_bug.cgi?id=480880
>  
>  Ian, I didn't have any luck with the noexec=off. 
It turns out that nonx_setup() and check_efer() both run quite a while
after all the set_page_prot calls in xen_setup_kernel() which include
_PAGE_NX via PAGE_KERNEL_RO.

On 32 bit __supported_pte_mask starts off without NX in it and it gets
added later if the system supports it. This is safe but means that the
pages frobbed by the early Xen setup won't have NX set when they could
(unless they all get frobbed again later?)

On 64 bit __supported_pte_mask contains NX at start of day and it is
taken away later on if the system turns out not to support it.

Native seems to mainly use _KERNPG_TABLE which does not include NX, can
you try this patch? (lots of printks because I don't have any non-NX
hardware to test properly).

diff -r ec792b22009f arch/x86/mm/init_64.c
--- a/arch/x86/mm/init_64.c	Fri Jan 23 15:27:45 2009 +0000
+++ b/arch/x86/mm/init_64.c	Fri Jan 23 15:58:03 2009 +0000
@@ -103,12 +103,15 @@
  */
 static int __init nonx_setup(char *str)  {
+	printk(KERN_CRIT "noexec_setup %s\n", str);
 	if (!str)
 		return -EINVAL;
 	if (!strncmp(str, "on", 2)) {
+		printk(KERN_CRIT "noexec_setup: enabling NX\n");
 		__supported_pte_mask |= _PAGE_NX;
 		do_not_nx = 0;
 	} else if (!strncmp(str, "off", 3)) {
+			printk(KERN_CRIT "noexec_setup: disabling
NX\n");
 		do_not_nx = 1;
 		__supported_pte_mask &= ~_PAGE_NX;
 	}
@@ -121,8 +124,13 @@
 	unsigned long efer;
 
 	rdmsrl(MSR_EFER, efer);
-	if (!(efer & EFER_NX) || do_not_nx)
+	if (!(efer & EFER_NX) || do_not_nx) {
+		printk(KERN_CRIT "check_efer: disabling NX\n");
 		__supported_pte_mask &= ~_PAGE_NX;
+	} else
+		printk(KERN_CRIT "check_efer: leaving NX alone.
supported_pte_mask %s the NX bit\n",
+		       __supported_pte_mask & _PAGE_NX ? "includes" :
"excludes");
+
 }
 
 int force_personality32;
diff -r ec792b22009f arch/x86/xen/enlighten.c
--- a/arch/x86/xen/enlighten.c	Fri Jan 23 15:27:45 2009 +0000
+++ b/arch/x86/xen/enlighten.c	Fri Jan 23 15:58:03 2009 +0000
@@ -54,6 +54,9 @@
 #include "xen-ops.h"
 #include "mmu.h"
 #include "multicalls.h"
+
+#define _KERNPG_TABLE_RO __pgprot(_KERNPG_TABLE & ~_PAGE_RW) //#define 
+_KERNPG_TABLE_RO (_KERNPG_TABLE)
 
 EXPORT_SYMBOL_GPL(hypercall_page);
 
@@ -1476,6 +1479,15 @@
 {
 	unsigned long pfn = __pa(addr) >> PAGE_SHIFT;
 	pte_t pte = pfn_pte(pfn, prot);
+	static int once = 5;
+
+	if (once > 0 && pte_val(pte) & _PAGE_NX) {
+		once--;
+		printk(KERN_CRIT "set_page_prot to %#lx (incl NX)
supported_pte_mask %#lx %s the NX bit\n",
+		       pgprot_val(prot), __supported_pte_mask,
__supported_pte_mask & _PAGE_NX ? "includes" : "excludes");
+		printk(KERN_CRIT "pte is %#lx\n", pte_val(pte));
+		WARN_ON(1);
+	}
 
 	if (HYPERVISOR_update_va_mapping((unsigned long)addr, pte, 0))
 		BUG();
@@ -1522,9 +1534,9 @@
 	}
 
 	for (pteidx = 0; pteidx < ident_pte; pteidx += PTRS_PER_PTE)
-		set_page_prot(&level1_ident_pgt[pteidx],
PAGE_KERNEL_RO);
+		set_page_prot(&level1_ident_pgt[pteidx],
_KERNPG_TABLE_RO);
 
-	set_page_prot(pmd, PAGE_KERNEL_RO);
+	set_page_prot(pmd, _KERNPG_TABLE_RO);
 }
 
 static __init void xen_ident_map_ISA(void) @@ -1601,12 +1613,12 @@
 	xen_map_identity_early(level2_ident_pgt, max_pfn);
 
 	/* Make pagetable pieces RO */
-	set_page_prot(init_level4_pgt, PAGE_KERNEL_RO);
-	set_page_prot(level3_ident_pgt, PAGE_KERNEL_RO);
-	set_page_prot(level3_kernel_pgt, PAGE_KERNEL_RO);
-	set_page_prot(level3_user_vsyscall, PAGE_KERNEL_RO);
-	set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO);
-	set_page_prot(level2_fixmap_pgt, PAGE_KERNEL_RO);
+	set_page_prot(init_level4_pgt, _KERNPG_TABLE_RO);
+	set_page_prot(level3_ident_pgt, _KERNPG_TABLE_RO);
+	set_page_prot(level3_kernel_pgt, _KERNPG_TABLE_RO);
+	set_page_prot(level3_user_vsyscall, _KERNPG_TABLE_RO);
+	set_page_prot(level2_kernel_pgt, _KERNPG_TABLE_RO);
+	set_page_prot(level2_fixmap_pgt, _KERNPG_TABLE_RO);
 
 	/* Pin down new L4 */
 	pin_pagetable_pfn(MMUEXT_PIN_L4_TABLE,
@@ -1670,9 +1682,9 @@
 	set_pgd(&swapper_pg_dir[KERNEL_PGD_BOUNDARY],
 			__pgd(__pa(level2_kernel_pgt) | _PAGE_PRESENT));
 
-	set_page_prot(level2_kernel_pgt, PAGE_KERNEL_RO);
-	set_page_prot(swapper_pg_dir, PAGE_KERNEL_RO);
-	set_page_prot(empty_zero_page, PAGE_KERNEL_RO);
+	set_page_prot(level2_kernel_pgt, _KERNPG_TABLE_RO);
+	set_page_prot(swapper_pg_dir, _KERNPG_TABLE_RO);
+	set_page_prot(empty_zero_page, _KERNPG_TABLE_RO);
 
 	pin_pagetable_pfn(MMUEXT_UNPIN_TABLE, PFN_DOWN(__pa(pgd)));