From d95142539827ce8d1f4650d7b565d7b02572cfe9 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Sat, 22 Jan 2022 10:56:33 +0100 Subject: [PATCH 1/6] pmm: fix locking in get_free_frames_cond() Signed-off-by: Pawel Wieczorkiewicz --- mm/pmm.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/mm/pmm.c b/mm/pmm.c index 70550a13..5a74fd52 100644 --- a/mm/pmm.c +++ b/mm/pmm.c @@ -376,17 +376,18 @@ static inline frame_t *return_frame(frame_t *frame) { * the condition specified by the callback */ frame_t *get_free_frames_cond(free_frames_cond_t cb) { - frame_t *frame; - spin_lock(&lock); for_each_order (order) { + frame_t *frame; + if (list_is_empty(&free_frames[order])) continue; list_for_each_entry (frame, &free_frames[order], list) { if (cb(frame)) { + reserve_frame(frame); spin_unlock(&lock); - return reserve_frame(frame); + return frame; } } } From 694c8ad5071754c790b3fd17a3a0ae190d39f47c Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Sat, 22 Jan 2022 11:02:08 +0100 Subject: [PATCH 2/6] pmm: replace put_frame() with destroy_frame() The destroy_frame() makes sure to unlink a frame from its list and update frames_count. Then it clears up frame's frame_array entry. Signed-off-by: Pawel Wieczorkiewicz --- mm/pmm.c | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/mm/pmm.c b/mm/pmm.c index 5a74fd52..f1968858 100644 --- a/mm/pmm.c +++ b/mm/pmm.c @@ -172,8 +172,8 @@ static inline frame_t *take_frame(frame_t *frame, frames_array_t *array) { return frame; } -static inline frame_t *put_frame(frame_t *frame, frames_array_t *array) { - ASSERT(!frame); +static inline frame_t *put_frames_array_entry(frame_t *frame, frames_array_t *array) { + BUG_ON(is_frame_free(frame)); if (!array) array = find_frames_array(frame); @@ -204,11 +204,15 @@ static inline frame_t *get_frames_array_entry(void) { return NULL; } -static inline void put_frames_array_entry(frame_t *frame) { - if (is_frame_free(frame)) - return; +static inline void destroy_frame(frame_t *frame) { + BUG_ON(is_frame_used(frame)); + + if (frame) { + list_unlink(&frame->list); + frames_count[frame->order]--; - put_frame(frame, NULL); + put_frames_array_entry(frame, NULL); + } } static inline frame_t *new_frame(mfn_t mfn, unsigned int order) { From 98c9ddbecce987de1e4fe623eb2dd9f50d12be94 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Sat, 22 Jan 2022 11:06:01 +0100 Subject: [PATCH 3/6] pmm: add find_mfn_frame() helper It allows to find a frame_t struct pointer for specified MFN and order. Signed-off-by: Pawel Wieczorkiewicz --- include/mm/pmm.h | 4 ++++ mm/pmm.c | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/include/mm/pmm.h b/include/mm/pmm.h index 96406e09..0f990bac 100644 --- a/include/mm/pmm.h +++ b/include/mm/pmm.h @@ -80,6 +80,10 @@ static inline bool paddr_invalid(paddr_t pa) { static inline bool mfn_invalid(mfn_t mfn) { return paddr_invalid(mfn_to_paddr(mfn)); } +static inline bool has_frames(list_head_t *frames, unsigned int order) { + return !(order > MAX_PAGE_ORDER || list_is_empty(&frames[order])); +} + static inline frame_t *get_free_frame(void) { return get_free_frames(PAGE_ORDER_4K); } static inline void put_free_frame(mfn_t mfn) { return put_free_frames(mfn, PAGE_ORDER_4K); diff --git a/mm/pmm.c b/mm/pmm.c index f1968858..fdec9d7a 100644 --- a/mm/pmm.c +++ b/mm/pmm.c @@ -376,6 +376,20 @@ static inline frame_t *return_frame(frame_t *frame) { return frame; } +static frame_t *find_mfn_frame(list_head_t *frames, mfn_t mfn, unsigned int order) { + frame_t *frame; + + if (!has_frames(frames, order)) + return NULL; + + list_for_each_entry (frame, &frames[order], list) { + if (frame->mfn == mfn) + return frame; + } + + return NULL; +} + /* Reserves and returns the first free frame fulfilling * the condition specified by the callback */ From 5e7002ed0d4d47bcdabffa49356e916e07bae84a Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Sat, 22 Jan 2022 11:14:55 +0100 Subject: [PATCH 4/6] pmm: add split_frame() functionality The split_frame() breaks given frame into two smaller frames. Original frame's frame_t struct is reused, frame order is decremented and the struct is re-assigned to lower order free frame list. Another, new frame is created for the second half of the original frame. The get_free_frame(), when cannot find free frame of requested order, keeps finding first free higher-order frame and split it, until desired order frame becomes available. Signed-off-by: Pawel Wieczorkiewicz --- include/mm/pmm.h | 10 +++++++++ mm/pmm.c | 54 ++++++++++++++++++++++++++++++++++++++++++------ 2 files changed, 58 insertions(+), 6 deletions(-) diff --git a/include/mm/pmm.h b/include/mm/pmm.h index 0f990bac..a352be93 100644 --- a/include/mm/pmm.h +++ b/include/mm/pmm.h @@ -60,6 +60,9 @@ typedef struct frames_array frames_array_t; typedef bool (*free_frames_cond_t)(frame_t *free_frame); +#define NEXT_MFN(mfn, order) ((mfn) + (1UL << (order))) +#define PREV_MFN(mfn, order) ((mfn) - (1UL << (order))) + /* External definitions */ extern void display_frames_count(void); @@ -84,6 +87,13 @@ static inline bool has_frames(list_head_t *frames, unsigned int order) { return !(order > MAX_PAGE_ORDER || list_is_empty(&frames[order])); } +static inline frame_t *get_first_frame(list_head_t *frames, unsigned int order) { + if (!has_frames(frames, order)) + return NULL; + + return list_first_entry(&frames[order], frame_t, list); +} + static inline frame_t *get_free_frame(void) { return get_free_frames(PAGE_ORDER_4K); } static inline void put_free_frame(mfn_t mfn) { return put_free_frames(mfn, PAGE_ORDER_4K); diff --git a/mm/pmm.c b/mm/pmm.c index fdec9d7a..37d88c1c 100644 --- a/mm/pmm.c +++ b/mm/pmm.c @@ -390,8 +390,20 @@ static frame_t *find_mfn_frame(list_head_t *frames, mfn_t mfn, unsigned int orde return NULL; } +static frame_t *find_larger_frame(list_head_t *frames, unsigned int order) { + while (++order <= MAX_PAGE_ORDER) { + frame_t *frame = get_first_frame(frames, order); + + if (frame) + return frame; + } + + return NULL; +} + /* Reserves and returns the first free frame fulfilling - * the condition specified by the callback + * the condition specified by the callback. + * This function does not split larger frames. */ frame_t *get_free_frames_cond(free_frames_cond_t cb) { spin_lock(&lock); @@ -414,6 +426,33 @@ frame_t *get_free_frames_cond(free_frames_cond_t cb) { return NULL; } +static inline void relink_frame_to_order(frame_t *frame, unsigned int new_order) { + BUG_ON(new_order > MAX_PAGE_ORDER); + + list_unlink(&frame->list); + frames_count[frame->order]--; + + frame->order = new_order; + + list_add_tail(&frame->list, &free_frames[frame->order]); + frames_count[frame->order]++; +} + +static void split_frame(frame_t *frame) { + BUG_ON(!frame); + + if (opt_debug) { + printk("PMM: Splitting frame:\n"); + display_frame(frame); + } + + /* First sibling frame */ + relink_frame_to_order(frame, frame->order - 1); + + /* Create new frame entry for the second sibling frame */ + add_frame(NEXT_MFN(frame->mfn, frame->order), frame->order); +} + frame_t *get_free_frames(unsigned int order) { frame_t *frame; @@ -421,13 +460,16 @@ frame_t *get_free_frames(unsigned int order) { return NULL; spin_lock(&lock); - if (list_is_empty(&free_frames[order])) { - /* FIXME: Add page split */ - spin_unlock(&lock); - return NULL; + while (list_is_empty(&free_frames[order])) { + frame = find_larger_frame(free_frames, order); + if (!frame) { + spin_unlock(&lock); + return NULL; + } + split_frame(frame); } - frame = reserve_frame(list_first_entry(&free_frames[order], frame_t, list)); + frame = reserve_frame(get_first_frame(free_frames, order)); spin_unlock(&lock); return frame; From a853cdad2402e33e29e6c1ae66ffee81e2515030 Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Sat, 22 Jan 2022 11:26:26 +0100 Subject: [PATCH 5/6] pmm: add merge_frames() functionality The merge_frames() attempts to merge two consecutive frames into a higher order frame. It only merges higher-order aligned frame with its following frame. To do so, it checks if the frame being returned is higher-order aligned frame and attempts to find its following frame. If the frame being returned is not higher-order aligned, it tries to find its preceding frame, which will be properly aligned. When the two frames are found, the first frame's struct is reused, order is incremented and the struct is relinked to higher order free frames list. The second frame struct is destroyed. The merge_frames() calls itself recursively in attempt to merge all merge- able higher-order frames for all orders. The put_free_frames() find corresponding frame struct for given MFN and order. If the frame is fully returned (there is no more references) and relinked from busy_frames to free_frames, it attempts to merge the frame. Signed-off-by: Pawel Wieczorkiewicz --- include/mm/pmm.h | 5 ++-- mm/pmm.c | 64 +++++++++++++++++++++++++++++++++++------------- 2 files changed, 50 insertions(+), 19 deletions(-) diff --git a/include/mm/pmm.h b/include/mm/pmm.h index a352be93..c4cbac7a 100644 --- a/include/mm/pmm.h +++ b/include/mm/pmm.h @@ -60,8 +60,9 @@ typedef struct frames_array frames_array_t; typedef bool (*free_frames_cond_t)(frame_t *free_frame); -#define NEXT_MFN(mfn, order) ((mfn) + (1UL << (order))) -#define PREV_MFN(mfn, order) ((mfn) - (1UL << (order))) +#define FIRST_FRAME_SIBLING(mfn, order) ((mfn) % (1UL << (order)) == 0) +#define NEXT_MFN(mfn, order) ((mfn) + (1UL << (order))) +#define PREV_MFN(mfn, order) ((mfn) - (1UL << (order))) /* External definitions */ diff --git a/mm/pmm.c b/mm/pmm.c index 37d88c1c..a9bc8025 100644 --- a/mm/pmm.c +++ b/mm/pmm.c @@ -364,16 +364,17 @@ static inline frame_t *reserve_frame(frame_t *frame) { return frame; } -static inline frame_t *return_frame(frame_t *frame) { +static inline bool return_frame(frame_t *frame) { if (!is_frame_used(frame)) panic("PMM: trying to return unused frame: %p\n", frame); if (--frame->refcount == 0) { list_unlink(&frame->list); list_add(&frame->list, &free_frames[frame->order]); + return true; } - return frame; + return false; } static frame_t *find_mfn_frame(list_head_t *frames, mfn_t mfn, unsigned int order) { @@ -453,6 +454,41 @@ static void split_frame(frame_t *frame) { add_frame(NEXT_MFN(frame->mfn, frame->order), frame->order); } +static void merge_frames(frame_t *first) { + frame_t *second; + + BUG_ON(!first); + + if (FIRST_FRAME_SIBLING(first->mfn, first->order + 1)) { + mfn_t next_mfn = NEXT_MFN(first->mfn, first->order); + second = find_mfn_frame(free_frames, next_mfn, first->order); + } + else { + /* Second frame sibling */ + mfn_t prev_mfn = PREV_MFN(first->mfn, first->order); + second = first; + first = find_mfn_frame(free_frames, prev_mfn, first->order); + } + + if (!first || !second) + return; + + if (opt_debug) { + printk("PMM: Merging frames:\n"); + display_frame(first); + display_frame(second); + } + + /* Make the first sibling a higher order frame */ + relink_frame_to_order(first, first->order + 1); + + /* Destroy the second sibling frame */ + destroy_frame(second); + + /* Try to merge higher order frames */ + merge_frames(first); +} + frame_t *get_free_frames(unsigned int order) { frame_t *frame; @@ -478,24 +514,18 @@ frame_t *get_free_frames(unsigned int order) { void put_free_frames(mfn_t mfn, unsigned int order) { frame_t *frame; - BUG_ON(mfn_invalid(mfn)); - - if (order > MAX_PAGE_ORDER) - return; + BUG_ON(mfn_invalid(mfn) || order > MAX_PAGE_ORDER); spin_lock(&lock); - list_for_each_entry (frame, &busy_frames[order], list) { - if (frame->mfn == mfn) { - /* FIXME: Maintain order wrt mfn value */ - /* FIXME: Add frame merge */ - return_frame(frame); - spin_unlock(&lock); - return; - } - } - spin_unlock(&lock); + frame = find_mfn_frame(busy_frames, mfn, order); + if (!frame) + panic("PMM: unable to find frame: %lx, order: %u among busy frames\n", mfn, + order); + + if (return_frame(frame)) + merge_frames(frame); - panic("PMM: unable to find frame: %x in busy frames list\n"); + spin_unlock(&lock); } void map_used_memory(void) { From 3b20ceb532ab229127ca71098f7eac80dc77624d Mon Sep 17 00:00:00 2001 From: Pawel Wieczorkiewicz Date: Sat, 22 Jan 2022 11:30:56 +0100 Subject: [PATCH 6/6] pmm: detect and use highest available frame order After creating early frames, the process_memory_range() finds highest available frame order (i.e. frame order whose size fits into available physical memory size). Instead of creating 2M frames by default, it creates frames of the detected order and uses the rest of available space to create 2M frames and 4K frames. Signed-off-by: Pawel Wieczorkiewicz --- include/mm/pmm.h | 2 ++ mm/pmm.c | 30 ++++++++++++++++++------------ 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/include/mm/pmm.h b/include/mm/pmm.h index c4cbac7a..f20c5ed5 100644 --- a/include/mm/pmm.h +++ b/include/mm/pmm.h @@ -60,6 +60,8 @@ typedef struct frames_array frames_array_t; typedef bool (*free_frames_cond_t)(frame_t *free_frame); +#define ORDER_TO_SIZE(order) (PAGE_SIZE << (order)) + #define FIRST_FRAME_SIBLING(mfn, order) ((mfn) % (1UL << (order)) == 0) #define NEXT_MFN(mfn, order) ((mfn) + (1UL << (order))) #define PREV_MFN(mfn, order) ((mfn) - (1UL << (order))) diff --git a/mm/pmm.c b/mm/pmm.c index a9bc8025..34d6f27d 100644 --- a/mm/pmm.c +++ b/mm/pmm.c @@ -237,8 +237,18 @@ static inline void add_frame(mfn_t mfn, unsigned int order) { list_add_tail(&frame->list, &free_frames[order]); } +static inline unsigned int find_max_avail_order(size_t size) { + for (unsigned int order = MAX_PAGE_ORDER; order > PAGE_ORDER_4K; order--) { + if (ORDER_TO_SIZE(order) <= size) + return order; + } + + return PAGE_ORDER_4K; +} + static size_t process_memory_range(unsigned index) { paddr_t start, end, cur; + unsigned int max_order; addr_range_t range; size_t size; @@ -261,31 +271,27 @@ static size_t process_memory_range(unsigned index) { add_early_frame(paddr_to_mfn(cur), PAGE_ORDER_4K); else add_frame(paddr_to_mfn(cur), PAGE_ORDER_4K); - cur += (PAGE_SIZE << PAGE_ORDER_4K); + cur += ORDER_TO_SIZE(PAGE_ORDER_4K); } - /* Add initial 2M frames and align to 1G. */ - while (cur % PAGE_SIZE_1G && cur + PAGE_SIZE_2M <= end) { - add_frame(paddr_to_mfn(cur), PAGE_ORDER_2M); - cur += (PAGE_SIZE << PAGE_ORDER_2M); - } + max_order = find_max_avail_order(end - cur); - /* Add all remaining 1G frames. */ - while (cur + PAGE_SIZE_1G <= end) { - add_frame(paddr_to_mfn(cur), PAGE_ORDER_1G); - cur += (PAGE_SIZE << PAGE_ORDER_1G); + /* Add all available max_order frames. */ + while (cur + ORDER_TO_SIZE(max_order) <= end) { + add_frame(paddr_to_mfn(cur), max_order); + cur += ORDER_TO_SIZE(max_order); } /* Add all remaining 2M frames. */ while (cur + PAGE_SIZE_2M <= end) { add_frame(paddr_to_mfn(cur), PAGE_ORDER_2M); - cur += (PAGE_SIZE << PAGE_ORDER_2M); + cur += ORDER_TO_SIZE(PAGE_ORDER_2M); } /* Add all remaining 4K frames. */ while (cur < end) { add_frame(paddr_to_mfn(cur), PAGE_ORDER_4K); - cur += (PAGE_SIZE << PAGE_ORDER_4K); + cur += ORDER_TO_SIZE(PAGE_ORDER_4K); } if (cur != end) {