From f889ed6dcbbdcc6b8904bbab62940e251261340e Mon Sep 17 00:00:00 2001
From: Pawel Wieczorkiewicz <wipawel@grsecurity.net>
Date: Fri, 8 Jul 2022 12:56:11 +0200
Subject: [PATCH 1/6] arch, pagetables: fix IS_ADDR_SPACE_VA() macro

The IS_ADDR_SPACE_VA() macro was using incorrect comparison for an
address belonging to an address space. Masking (&) could return
incorrect result for a shifted user address space mappings.

Signed-off-by: Pawel Wieczorkiewicz <wipawel@grsecurity.net>
---
 include/arch/x86/page.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/include/arch/x86/page.h b/include/arch/x86/page.h
index 2a324579..77aa9741 100644
--- a/include/arch/x86/page.h
+++ b/include/arch/x86/page.h
@@ -153,7 +153,7 @@ typedef unsigned long mfn_t;
 #define PADDR_INVALID (0UL)
 #define MFN_INVALID   (0UL)
 
-#define IS_ADDR_SPACE_VA(va, as) ((_ul(va) & (as)) == (as))
+#define IS_ADDR_SPACE_VA(va, as) (_ul(va) >= (as))
 
 /* External declarations */
 
@@ -210,6 +210,7 @@ static inline void *mfn_to_virt(mfn_t mfn) { return paddr_to_virt(mfn << PAGE_SH
 static inline paddr_t virt_to_paddr(const void *va) {
     paddr_t pa = (paddr_t) va;
 
+    /* Order matters here */
     if (IS_ADDR_SPACE_VA(va, VIRT_KERNEL_BASE))
         return pa - VIRT_KERNEL_BASE;
     if (IS_ADDR_SPACE_VA(va, VIRT_KERNEL_MAP))

From 3a6c6e24a6bf6c1d5b8500e3c9c1f657636634a9 Mon Sep 17 00:00:00 2001
From: Pawel Wieczorkiewicz <wipawel@grsecurity.net>
Date: Fri, 8 Jul 2022 12:58:13 +0200
Subject: [PATCH 2/6] arch, pagetables: fix vmap()'s initial va alignment check

It should check VA alignment based on requested order, not just
PAGE_MASK (4K pages order).

Signed-off-by: Pawel Wieczorkiewicz <wipawel@grsecurity.net>
---
 arch/x86/pagetables.c   | 2 +-
 include/arch/x86/page.h | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/arch/x86/pagetables.c b/arch/x86/pagetables.c
index cf3da855..58761dc6 100644
--- a/arch/x86/pagetables.c
+++ b/arch/x86/pagetables.c
@@ -161,7 +161,7 @@ void *vmap(void *va, mfn_t mfn, unsigned int order,
     mfn_t l1t_mfn, l2t_mfn, l3t_mfn;
     pgentry_t *tab, *entry;
 
-    if (!va || _ul(va) & ~PAGE_MASK)
+    if (!va || (_ul(va) & ~PAGE_ORDER_TO_MASK(order)))
         return NULL;
 
     dprintk("%s: va: %p mfn: 0x%lx (order: %u)\n", __func__, va, mfn, order);
diff --git a/include/arch/x86/page.h b/include/arch/x86/page.h
index 77aa9741..c7bc16de 100644
--- a/include/arch/x86/page.h
+++ b/include/arch/x86/page.h
@@ -45,6 +45,8 @@
 #define MAX_PAGE_ORDER     PAGE_ORDER_1G
 #define PAGE_ORDER_INVALID (-1)
 
+#define PAGE_ORDER_TO_MASK(order) (~((PAGE_SIZE << (order)) - 1))
+
 #define _PAGE_PRESENT  0x0001
 #define _PAGE_RW       0x0002
 #define _PAGE_USER     0x0004

From d2a9b3ab7d53d305035cb0107d3ba6bc0a8304fe Mon Sep 17 00:00:00 2001
From: Pawel Wieczorkiewicz <wipawel@grsecurity.net>
Date: Fri, 8 Jul 2022 13:23:24 +0200
Subject: [PATCH 3/6] arch, pagetables: prepare vmap() to handle separate
 address spaces

Make _vmap() accept CR3 as parameter, indicating which address space
is being mapped into. And turn vmap() and vunmap() into helper functions
with default address space of the kernel (vmap_kern(), vunmap_kern()).

Remove mmap_*() helper functions as they are confusing and not used.

Signed-off-by: Pawel Wieczorkiewicz <wipawel@grsecurity.net>
---
 arch/x86/pagetables.c     | 20 +++++++++-----
 common/setup.c            |  2 +-
 drivers/acpi/acpica/osl.c |  6 ++---
 include/arch/x86/page.h   | 55 ++++++++++++---------------------------
 mm/vmm.c                  | 27 ++++++++++++-------
 5 files changed, 50 insertions(+), 60 deletions(-)

diff --git a/arch/x86/pagetables.c b/arch/x86/pagetables.c
index 58761dc6..2cec2051 100644
--- a/arch/x86/pagetables.c
+++ b/arch/x86/pagetables.c
@@ -152,11 +152,11 @@ static mfn_t get_pgentry_mfn(mfn_t tab_mfn, pt_index_t index, unsigned long flag
     return mfn;
 }
 
-void *vmap(void *va, mfn_t mfn, unsigned int order,
+void *_vmap(cr3_t *cr3, void *va, mfn_t mfn, unsigned int order,
 #if defined(__x86_64__)
-           unsigned long l4_flags,
+            unsigned long l4_flags,
 #endif
-           unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags) {
+            unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags) {
     static spinlock_t lock = SPINLOCK_INIT;
     mfn_t l1t_mfn, l2t_mfn, l3t_mfn;
     pgentry_t *tab, *entry;
@@ -169,9 +169,9 @@ void *vmap(void *va, mfn_t mfn, unsigned int order,
     spin_lock(&lock);
 
 #if defined(__x86_64__)
-    l3t_mfn = get_pgentry_mfn(get_cr3_mfn(&cr3), l4_table_index(va), l4_flags);
+    l3t_mfn = get_pgentry_mfn(get_cr3_mfn(cr3), l4_table_index(va), l4_flags);
 #else
-    l3t_mfn = get_cr3_mfn(&cr3);
+    l3t_mfn = get_cr3_mfn(cr3);
 #endif
 
     if (order == PAGE_ORDER_1G) {
@@ -201,8 +201,14 @@ void *vmap(void *va, mfn_t mfn, unsigned int order,
     return va;
 }
 
-void vunmap(void *va, unsigned int order) {
-    vmap(va, MFN_INVALID, order, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS);
+void *vmap_kern(void *va, mfn_t mfn, unsigned int order,
+#if defined(__x86_64__)
+                unsigned long l4_flags,
+#endif
+                unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags) {
+    unsigned long _va = _ul(va) & PAGE_ORDER_TO_MASK(order);
+
+    return _vmap(&cr3, _ptr(_va), mfn, order, l4_flags, l3_flags, l2_flags, l1_flags);
 }
 
 void init_pagetables(void) {
diff --git a/common/setup.c b/common/setup.c
index 4eb74006..42cbdaf9 100644
--- a/common/setup.c
+++ b/common/setup.c
@@ -105,7 +105,7 @@ void zap_boot_mappings(void) {
                 memset(r->start, 0, r->end - r->start);
 
             for (mfn_t mfn = virt_to_mfn(r->start); mfn < virt_to_mfn(r->end); mfn++) {
-                vunmap(mfn_to_virt(mfn), PAGE_ORDER_4K);
+                vunmap_kern(mfn_to_virt(mfn), PAGE_ORDER_4K);
                 reclaim_frame(mfn, PAGE_ORDER_4K);
             }
         }
diff --git a/drivers/acpi/acpica/osl.c b/drivers/acpi/acpica/osl.c
index 9d372a00..23511159 100644
--- a/drivers/acpi/acpica/osl.c
+++ b/drivers/acpi/acpica/osl.c
@@ -294,7 +294,8 @@ void *AcpiOsMapMemory(ACPI_PHYSICAL_ADDRESS PhysicalAddress, ACPI_SIZE Length) {
         void *_va;
 
         if (!frame) {
-            _va = mmap_4k(mfn, L1_PROT);
+            _va = vmap_kern(mfn_to_virt_map(mfn), mfn, PAGE_ORDER_4K, L4_PROT, L3_PROT,
+                            L2_PROT, L1_PROT);
             if (!_va) {
                 spin_unlock(&map_lock);
                 return NULL;
@@ -327,7 +328,7 @@ void AcpiOsUnmapMemory(void *LogicalAddress, ACPI_SIZE Length) {
         if (--frame->refcount > 0)
             continue;
 
-        vunmap(mfn_to_virt_map(mfn), PAGE_ORDER_4K);
+        vunmap_kern(mfn_to_virt_map(mfn), PAGE_ORDER_4K);
         list_unlink(&frame->list);
         kfree(frame);
     }
@@ -351,7 +352,6 @@ typedef struct osd_exec_cb_wrapper osd_exec_cb_wrapper_t;
 
 unsigned long _osd_exec_cb_wrapper(void *arg) {
     osd_exec_cb_wrapper_t *cb = arg;
-
     cb->Function(cb->Context);
     return 0;
 }
diff --git a/include/arch/x86/page.h b/include/arch/x86/page.h
index c7bc16de..468234d6 100644
--- a/include/arch/x86/page.h
+++ b/include/arch/x86/page.h
@@ -159,13 +159,13 @@ typedef unsigned long mfn_t;
 
 /* External declarations */
 
-extern void *vmap(void *va, mfn_t mfn, unsigned int order,
+extern void *vmap_kern(void *va, mfn_t mfn, unsigned int order,
 #if defined(__x86_64__)
-                  unsigned long l4_flags,
+                       unsigned long l4_flags,
 #endif
-                  unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags);
+                       unsigned long l3_flags, unsigned long l2_flags,
+                       unsigned long l1_flags);
 
-extern void vunmap(void *va, unsigned int order);
 
 extern void pat_set_type(pat_field_t field, pat_memory_type_t type);
 extern pat_memory_type_t pat_get_type(pat_field_t field);
@@ -227,68 +227,45 @@ static inline mfn_t virt_to_mfn(const void *va) {
     return paddr_to_mfn(virt_to_paddr(va));
 }
 
-static inline void *kmap(mfn_t mfn, unsigned int order,
-#if defined(__x86_64__)
-                         unsigned long l4_flags,
-#endif
-                         unsigned long l3_flags, unsigned long l2_flags,
-                         unsigned long l1_flags) {
-    return vmap(mfn_to_virt_kern(mfn), mfn, order,
-#if defined(__x86_64__)
-                l4_flags,
-#endif
-                l3_flags, l2_flags, l1_flags);
+static inline void vunmap_kern(void *va, unsigned int order) {
+    vmap_kern(va, MFN_INVALID, order, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS);
 }
 
-static inline void *mmap(mfn_t mfn, unsigned int order,
+static inline void *kmap(mfn_t mfn, unsigned int order,
 #if defined(__x86_64__)
                          unsigned long l4_flags,
 #endif
                          unsigned long l3_flags, unsigned long l2_flags,
                          unsigned long l1_flags) {
-    return vmap(mfn_to_virt_map(mfn), mfn, order,
+    return vmap_kern(mfn_to_virt_kern(mfn), mfn, order,
 #if defined(__x86_64__)
-                l4_flags,
+                     l4_flags,
 #endif
-                l3_flags, l2_flags, l1_flags);
+                     l3_flags, l2_flags, l1_flags);
 }
 
 static inline void *vmap_1g(void *va, mfn_t mfn, unsigned long l3_flags) {
-    return vmap(va, mfn, PAGE_ORDER_1G, L4_PROT_USER, l3_flags, PT_NO_FLAGS, PT_NO_FLAGS);
+    return vmap_kern(va, mfn, PAGE_ORDER_1G, L4_PROT, l3_flags, PT_NO_FLAGS, PT_NO_FLAGS);
 }
 
 static inline void *vmap_2m(void *va, mfn_t mfn, unsigned long l2_flags) {
-    return vmap(va, mfn, PAGE_ORDER_2M, L4_PROT_USER, L3_PROT_USER, l2_flags,
-                PT_NO_FLAGS);
+    return vmap_kern(va, mfn, PAGE_ORDER_2M, L4_PROT, L3_PROT, l2_flags, PT_NO_FLAGS);
 }
 
 static inline void *vmap_4k(void *va, mfn_t mfn, unsigned long l1_flags) {
-    return vmap(va, mfn, PAGE_ORDER_4K, L4_PROT_USER, L3_PROT_USER, L2_PROT_USER,
-                l1_flags);
+    return vmap_kern(va, mfn, PAGE_ORDER_4K, L4_PROT, L3_PROT, L2_PROT, l1_flags);
 }
 
 static inline void *kmap_1g(mfn_t mfn, unsigned long l3_flags) {
-    return kmap(mfn, PAGE_ORDER_1G, L4_PROT_USER, l3_flags, PT_NO_FLAGS, PT_NO_FLAGS);
+    return kmap(mfn, PAGE_ORDER_1G, L4_PROT, l3_flags, PT_NO_FLAGS, PT_NO_FLAGS);
 }
 
 static inline void *kmap_2m(mfn_t mfn, unsigned long l2_flags) {
-    return kmap(mfn, PAGE_ORDER_2M, L4_PROT_USER, L3_PROT_USER, l2_flags, PT_NO_FLAGS);
+    return kmap(mfn, PAGE_ORDER_2M, L4_PROT, L3_PROT, l2_flags, PT_NO_FLAGS);
 }
 
 static inline void *kmap_4k(mfn_t mfn, unsigned long l1_flags) {
-    return kmap(mfn, PAGE_ORDER_4K, L4_PROT_USER, L3_PROT_USER, L2_PROT_USER, l1_flags);
-}
-
-static inline void *mmap_1g(mfn_t mfn, unsigned long l3_flags) {
-    return mmap(mfn, PAGE_ORDER_1G, L4_PROT_USER, l3_flags, PT_NO_FLAGS, PT_NO_FLAGS);
-}
-
-static inline void *mmap_2m(mfn_t mfn, unsigned long l2_flags) {
-    return mmap(mfn, PAGE_ORDER_2M, L4_PROT_USER, L3_PROT_USER, l2_flags, PT_NO_FLAGS);
-}
-
-static inline void *mmap_4k(mfn_t mfn, unsigned long l1_flags) {
-    return mmap(mfn, PAGE_ORDER_4K, L4_PROT_USER, L3_PROT_USER, L2_PROT_USER, l1_flags);
+    return kmap(mfn, PAGE_ORDER_4K, L4_PROT, L3_PROT, L2_PROT, l1_flags);
 }
 
 #endif /* __ASSEMBLY__ */
diff --git a/mm/vmm.c b/mm/vmm.c
index 6d86b21d..2983275f 100644
--- a/mm/vmm.c
+++ b/mm/vmm.c
@@ -40,24 +40,31 @@ void *get_free_pages(unsigned int order, uint32_t flags) {
 
     if (!frame)
         return NULL;
-
     mfn = frame->mfn;
 
-    if (flags & GFP_IDENT)
-        va = vmap(mfn_to_virt(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT, L1_PROT);
-    if (flags & GFP_USER)
-        va = vmap(mfn_to_virt_user(mfn), mfn, order, L4_PROT_USER, L3_PROT_USER,
-                  L2_PROT_USER, L1_PROT_USER);
-    if (flags & GFP_KERNEL)
+    if (flags == GFP_USER) {
+        va = vmap_kern(mfn_to_virt_user(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT,
+                       L1_PROT);
+    }
+
+    if (flags & GFP_IDENT) {
+        va = vmap_kern(mfn_to_virt(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT, L1_PROT);
+    }
+
+    if (flags & GFP_KERNEL) {
         va = kmap(mfn, order, L4_PROT, L3_PROT, L2_PROT, L1_PROT);
-    if (flags & GFP_KERNEL_MAP)
-        va = mmap(mfn, order, L4_PROT, L3_PROT, L2_PROT, L1_PROT);
+    }
+
+    if (flags & GFP_KERNEL_MAP) {
+        va = vmap_kern(mfn_to_virt_map(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT,
+                       L1_PROT);
+    }
 
     return va;
 }
 
 void put_pages(void *page, unsigned int order) {
     /* FIXME: unmap all mappings */
-    vunmap(page, order);
+    vunmap_kern(page, order);
     put_free_frames(virt_to_mfn(page), order);
 }

From 501b6e720a36e8f20de5fe81a7a0af3940cd6d16 Mon Sep 17 00:00:00 2001
From: Pawel Wieczorkiewicz <wipawel@grsecurity.net>
Date: Fri, 8 Jul 2022 13:24:10 +0200
Subject: [PATCH 4/6] mm/vmm: add put_page_top() helper function for stack
 pages

Signed-off-by: Pawel Wieczorkiewicz <wipawel@grsecurity.net>
---
 include/mm/vmm.h | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/include/mm/vmm.h b/include/mm/vmm.h
index 80f68a17..e8e48c21 100644
--- a/include/mm/vmm.h
+++ b/include/mm/vmm.h
@@ -55,4 +55,8 @@ static inline void *get_free_page_top(uint32_t flags) {
 
 static inline void put_page(void *page) { put_pages(page, PAGE_ORDER_4K); }
 
+static inline void put_page_top(void *page) {
+    put_pages(page - PAGE_SIZE, PAGE_ORDER_4K);
+}
+
 #endif /* KTF_VMM_H */

From 15d004d82e335106b79497606b49187acbfc204a Mon Sep 17 00:00:00 2001
From: Pawel Wieczorkiewicz <wipawel@grsecurity.net>
Date: Fri, 8 Jul 2022 13:55:42 +0200
Subject: [PATCH 5/6] arch, pagetables: add user address space

Add separate CR3 variable holding top of the user space page tables.
Add vmap_user*() helper functions, capable of mapping both kernel
mappings (kernel privilege, for necessary kernel mappings in user
address space) and user mappings (userland data and code).

Map userland data and code to both kernel and user address spaces.

Map IDT and TSS's RSP0 stack to user address space as kernel mappings.

Signed-off-by: Pawel Wieczorkiewicz <wipawel@grsecurity.net>
---
 arch/x86/pagetables.c        | 20 ++++++++++++++++++--
 arch/x86/traps.c             |  4 ++--
 include/arch/x86/page.h      | 31 +++++++++++++++++++++++++++++++
 include/arch/x86/pagetable.h |  1 +
 mm/vmm.c                     | 10 ++++++++++
 5 files changed, 62 insertions(+), 4 deletions(-)

diff --git a/arch/x86/pagetables.c b/arch/x86/pagetables.c
index 2cec2051..3088e0c2 100644
--- a/arch/x86/pagetables.c
+++ b/arch/x86/pagetables.c
@@ -31,6 +31,7 @@
 #include <string.h>
 
 cr3_t cr3;
+cr3_t user_cr3;
 
 static inline const char *dump_pte_flags(char *buf, size_t size, pte_t pte) {
     /* clang-format off */
@@ -211,6 +212,17 @@ void *vmap_kern(void *va, mfn_t mfn, unsigned int order,
     return _vmap(&cr3, _ptr(_va), mfn, order, l4_flags, l3_flags, l2_flags, l1_flags);
 }
 
+void *vmap_user(void *va, mfn_t mfn, unsigned int order,
+#if defined(__x86_64__)
+                unsigned long l4_flags,
+#endif
+                unsigned long l3_flags, unsigned long l2_flags, unsigned long l1_flags) {
+    unsigned long _va = _ul(va) & PAGE_ORDER_TO_MASK(order);
+
+    return _vmap(&user_cr3, _ptr(_va), mfn, order, l4_flags, l3_flags, l2_flags,
+                 l1_flags);
+}
+
 void init_pagetables(void) {
     for_each_memory_range (r) {
         switch (r->base) {
@@ -223,8 +235,12 @@ void init_pagetables(void) {
                 kmap_4k(mfn, r->flags);
             break;
         case VIRT_USER_BASE:
-            for (mfn_t mfn = virt_to_mfn(r->start); mfn < virt_to_mfn(r->end); mfn++)
-                vmap_4k(mfn_to_virt_user(mfn), mfn, r->flags);
+            for (mfn_t mfn = virt_to_mfn(r->start); mfn < virt_to_mfn(r->end); mfn++) {
+                void *va = mfn_to_virt_user(mfn);
+
+                vmap_4k(va, mfn, r->flags);
+                vmap_user_4k(va, mfn, r->flags);
+            }
             break;
         default:
             break;
diff --git a/arch/x86/traps.c b/arch/x86/traps.c
index 5791a6c0..f2733d3e 100644
--- a/arch/x86/traps.c
+++ b/arch/x86/traps.c
@@ -77,7 +77,7 @@ static void init_tss(percpu_t *percpu) {
     percpu->tss.ss0 = __KERN_DS;
     percpu->tss.cr3 = _ul(cr3.reg);
 #elif defined(__x86_64__)
-    percpu->tss.rsp0 = _ul(get_free_page_top(GFP_KERNEL));
+    percpu->tss.rsp0 = _ul(get_free_page_top(GFP_KERNEL | GFP_USER));
     percpu->tss.ist[0] = _ul(get_free_page_top(GFP_KERNEL));
 #endif
     percpu->tss.iopb = sizeof(percpu->tss);
@@ -124,7 +124,7 @@ void init_traps(unsigned int cpu) {
 
     BUG_ON(!percpu);
 
-    percpu->idt = get_free_page(GFP_KERNEL);
+    percpu->idt = get_free_page(GFP_KERNEL | GFP_USER);
     BUG_ON(!percpu->idt);
 
     percpu->idt_ptr.size = (sizeof(percpu->idt) * MAX_INT) - 1;
diff --git a/include/arch/x86/page.h b/include/arch/x86/page.h
index 468234d6..d9206398 100644
--- a/include/arch/x86/page.h
+++ b/include/arch/x86/page.h
@@ -166,6 +166,12 @@ extern void *vmap_kern(void *va, mfn_t mfn, unsigned int order,
                        unsigned long l3_flags, unsigned long l2_flags,
                        unsigned long l1_flags);
 
+extern void *vmap_user(void *va, mfn_t mfn, unsigned int order,
+#if defined(__x86_64__)
+                       unsigned long l4_flags,
+#endif
+                       unsigned long l3_flags, unsigned long l2_flags,
+                       unsigned long l1_flags);
 
 extern void pat_set_type(pat_field_t field, pat_memory_type_t type);
 extern pat_memory_type_t pat_get_type(pat_field_t field);
@@ -231,6 +237,10 @@ static inline void vunmap_kern(void *va, unsigned int order) {
     vmap_kern(va, MFN_INVALID, order, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS);
 }
 
+static inline void vunmap_user(void *va, unsigned int order) {
+    vmap_user(va, MFN_INVALID, order, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS, PT_NO_FLAGS);
+}
+
 static inline void *kmap(mfn_t mfn, unsigned int order,
 #if defined(__x86_64__)
                          unsigned long l4_flags,
@@ -268,6 +278,27 @@ static inline void *kmap_4k(mfn_t mfn, unsigned long l1_flags) {
     return kmap(mfn, PAGE_ORDER_4K, L4_PROT, L3_PROT, L2_PROT, l1_flags);
 }
 
+static inline void *vmap_user_1g(void *va, mfn_t mfn, unsigned long l3_flags) {
+    unsigned long user = l3_flags & _PAGE_USER;
+
+    return vmap_user(va, mfn, PAGE_ORDER_1G, L4_PROT | user, l3_flags | user, PT_NO_FLAGS,
+                     PT_NO_FLAGS);
+}
+
+static inline void *vmap_user_2m(void *va, mfn_t mfn, unsigned long l2_flags) {
+    unsigned long user = l2_flags & _PAGE_USER;
+
+    return vmap_user(va, mfn, PAGE_ORDER_2M, L4_PROT | user, L3_PROT | user,
+                     l2_flags | user, PT_NO_FLAGS);
+}
+
+static inline void *vmap_user_4k(void *va, mfn_t mfn, unsigned long l1_flags) {
+    unsigned long user = l1_flags & _PAGE_USER;
+
+    return vmap_user(va, mfn, PAGE_ORDER_4K, L4_PROT | user, L3_PROT | user,
+                     L2_PROT | user, l1_flags);
+}
+
 #endif /* __ASSEMBLY__ */
 
 #endif /* KTF_PAGE_H */
diff --git a/include/arch/x86/pagetable.h b/include/arch/x86/pagetable.h
index e70eb2a7..3d7f7866 100644
--- a/include/arch/x86/pagetable.h
+++ b/include/arch/x86/pagetable.h
@@ -137,6 +137,7 @@ union cr3 {
 typedef union cr3 cr3_t;
 
 extern cr3_t cr3;
+extern cr3_t user_cr3;
 
 typedef unsigned int pt_index_t;
 
diff --git a/mm/vmm.c b/mm/vmm.c
index 2983275f..fad57929 100644
--- a/mm/vmm.c
+++ b/mm/vmm.c
@@ -45,19 +45,29 @@ void *get_free_pages(unsigned int order, uint32_t flags) {
     if (flags == GFP_USER) {
         va = vmap_kern(mfn_to_virt_user(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT,
                        L1_PROT);
+        vmap_user(mfn_to_virt_user(mfn), mfn, order, L4_PROT_USER, L3_PROT_USER,
+                  L2_PROT_USER, L1_PROT_USER);
     }
 
     if (flags & GFP_IDENT) {
         va = vmap_kern(mfn_to_virt(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT, L1_PROT);
+        if (flags & GFP_USER)
+            vmap_user(mfn_to_virt(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT, L1_PROT);
     }
 
     if (flags & GFP_KERNEL) {
         va = kmap(mfn, order, L4_PROT, L3_PROT, L2_PROT, L1_PROT);
+        if (flags & GFP_USER)
+            vmap_user(mfn_to_virt_kern(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT,
+                      L1_PROT);
     }
 
     if (flags & GFP_KERNEL_MAP) {
         va = vmap_kern(mfn_to_virt_map(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT,
                        L1_PROT);
+        if (flags & GFP_USER)
+            vmap_user(mfn_to_virt_map(mfn), mfn, order, L4_PROT, L3_PROT, L2_PROT,
+                      L1_PROT);
     }
 
     return va;

From 201f5d3d52f0449d2ac2787fd2975d115d553bee Mon Sep 17 00:00:00 2001
From: Pawel Wieczorkiewicz <wipawel@grsecurity.net>
Date: Fri, 8 Jul 2022 13:57:36 +0200
Subject: [PATCH 6/6] arch: add SET_CR3 assembly macro

The macro preserves RAX register via stack, but does not clean the stack.

Signed-off-by: Pawel Wieczorkiewicz <wipawel@grsecurity.net>
---
 include/arch/x86/asm-macros.h | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/include/arch/x86/asm-macros.h b/include/arch/x86/asm-macros.h
index 6b946d57..acdb7edd 100644
--- a/include/arch/x86/asm-macros.h
+++ b/include/arch/x86/asm-macros.h
@@ -152,6 +152,13 @@
 #endif
 .endm
 
+.macro SET_CR3 val
+    push %_ASM_AX
+    mov (\val), %_ASM_AX
+    mov %_ASM_AX, %cr3
+    pop %_ASM_AX
+.endm
+
 #define GLOBAL(name) \
     .global name;    \
 name: