diff -Naur /home/rni/linux-2.6.12-rc5/include/linux/cart.h linux-cart.v3/include/linux/cart.h --- /home/rni/linux-2.6.12-rc5/include/linux/cart.h 1969-12-31 19:00:00.000000000 -0500 +++ linux-cart.v3/include/linux/cart.h 2005-08-16 14:01:20.000000000 -0400 @@ -0,0 +1,28 @@ +#ifndef __CART_H__ +#define __CART_H__ +#include +#include + +#define MIN_POOL 512 + +#define EVICTED_ACTIVE 1 +#define EVICTED_LONGTERM 2 +#define ACTIVE 4 +#define ACTIVE_LONGTERM 8 + +#define EvictedActive(location) (location & EVICTED_ACTIVE) +#define EvictedLongterm(location) (location & EVICTED_LONGTERM) +#define Active(location) (location & ACTIVE) +#define ActiveLongterm(location) (location & ACTIVE_LONGTERM) + +struct non_res_list_node { + struct list_head list; + struct list_head hash; + unsigned long hashval; +}; + +extern void cart_init(); +void update_cart_params(struct page *); +struct page *replace(struct zone *, int *); +#endif + diff -Naur /home/rni/linux-2.6.12-rc5/include/linux/evicted_hash.h linux-cart.v3/include/linux/evicted_hash.h --- /home/rni/linux-2.6.12-rc5/include/linux/evicted_hash.h 1969-12-31 19:00:00.000000000 -0500 +++ linux-cart.v3/include/linux/evicted_hash.h 2005-08-16 14:06:54.000000000 -0400 @@ -0,0 +1,10 @@ +#include +#include +#include + +void hashtable_init(struct hashtable *h); +unsigned long mk_hash_page(struct page *page); +unsigned long find_in_hashtable(struct hashtable *h, struct page *page); +void add_to_hashtable (struct hashtable *, struct non_res_list_node *); +unsigned long get_inode_num(void *addr); + diff -Naur /home/rni/linux-2.6.12-rc5/include/linux/mm_inline.h linux-cart.v3/include/linux/mm_inline.h --- /home/rni/linux-2.6.12-rc5/include/linux/mm_inline.h 2005-05-24 23:31:20.000000000 -0400 +++ linux-cart.v3/include/linux/mm_inline.h 2005-08-16 13:59:51.000000000 -0400 @@ -38,3 +38,32 @@ zone->nr_inactive--; } } + +static inline void +add_page_to_active_list_tail(struct zone *zone, struct page *page) +{ + list_add_tail(&page->lru, &zone->active_list); + zone->nr_active++; +} + +static inline void +add_page_to_inactive_list_tail(struct zone *zone, struct page *page) +{ + list_add_tail(&page->lru, &zone->inactive_list); + zone->nr_inactive++; +} + +static inline void +del_page_from_active_longterm(struct zone *zone, struct page *page) +{ + list_del(&page->lru); + zone->nr_active_longterm--; +} + +static inline void +add_page_to_active_longterm_tail(struct zone *zone, struct page *page) +{ + list_add_tail(&page->lru, &zone->active_longterm); + zone->nr_active_longterm++; +} + diff -Naur /home/rni/linux-2.6.12-rc5/include/linux/mmzone.h linux-cart.v3/include/linux/mmzone.h --- /home/rni/linux-2.6.12-rc5/include/linux/mmzone.h 2005-05-24 23:31:20.000000000 -0400 +++ linux-cart.v3/include/linux/mmzone.h 2005-08-16 13:59:50.000000000 -0400 @@ -26,6 +26,18 @@ unsigned long nr_free; }; +/* A Hashtable for the evicted list entries + * This is here, and not in linux/evicted_hash.h + * as the hashtable is needed in struct zone, but + * the function prototypes for the hash table use + * struct page, which is as yet unrecognised here + */ +#define HASHTABLE_SIZE 512 + +struct hashtable { + struct list_head buckets[HASHTABLE_SIZE]; +}; + struct pglist_data; /* @@ -135,12 +147,24 @@ /* Fields commonly accessed by the page reclaim scanner */ spinlock_t lru_lock; - struct list_head active_list; + struct list_head active_list; /* The T1 list of CART */ + struct list_head active_longterm;/* The T2 list of CART */ struct list_head inactive_list; + struct list_head evicted_active; /* The B1 list of CART */ + struct list_head evicted_longterm;/*B2 list of CART */ + struct hashtable evicted_active_hashtable; /*Hash table for evicted active list */ + struct hashtable evicted_longterm_hashtable; /*Hast table for evicted inactive list */ unsigned long nr_scan_active; unsigned long nr_scan_inactive; - unsigned long nr_active; + unsigned long nr_active; + unsigned long nr_active_longterm; unsigned long nr_inactive; + unsigned long nr_evicted_active; + unsigned long nr_evicted_longterm; + unsigned long nr_longterm; /* number of long term pages */ + unsigned long nr_shortterm; /* number of short term pages */ + unsigned long p; /* p from the CART paper */ + unsigned long q; /* q from the cart paper */ unsigned long pages_scanned; /* since last reclaim */ int all_unreclaimable; /* All pages pinned */ diff -Naur /home/rni/linux-2.6.12-rc5/include/linux/page-flags.h linux-cart.v3/include/linux/page-flags.h --- /home/rni/linux-2.6.12-rc5/include/linux/page-flags.h 2005-05-24 23:31:20.000000000 -0400 +++ linux-cart.v3/include/linux/page-flags.h 2005-08-16 13:59:51.000000000 -0400 @@ -58,7 +58,7 @@ #define PG_dirty 4 #define PG_lru 5 -#define PG_active 6 +#define PG_active 6 #define PG_slab 7 /* slab debug (Suparna wants this) */ #define PG_highmem 8 @@ -76,6 +76,7 @@ #define PG_reclaim 18 /* To be reclaimed asap */ #define PG_nosave_free 19 /* Free, should not be written */ #define PG_uncached 20 /* Page has been mapped as uncached */ +#define PG_longterm 21 /* Filter bit for CART see mm/cart.c */ /* * Global page accounting. One instance per CPU. Only unsigned longs are @@ -306,6 +307,10 @@ #define SetPageUncached(page) set_bit(PG_uncached, &(page)->flags) #define ClearPageUncached(page) clear_bit(PG_uncached, &(page)->flags) +#define PageLongTerm(page) test_bit(PG_longterm, &(page)->flags) +#define SetLongTerm(page) set_bit(PG_longterm, &(page)->flags) +#define ClearLongTerm(page) clear_bit(PG_longterm, &(page)->flags) + struct page; /* forward declaration */ int test_clear_page_dirty(struct page *page); diff -Naur /home/rni/linux-2.6.12-rc5/init/main.c linux-cart.v3/init/main.c --- /home/rni/linux-2.6.12-rc5/init/main.c 2005-05-24 23:31:20.000000000 -0400 +++ linux-cart.v3/init/main.c 2005-08-16 13:59:51.000000000 -0400 @@ -52,6 +52,8 @@ #include #include +#include + /* * This is one of the first .c files built. Error out early * if we have compiler trouble.. @@ -490,6 +492,7 @@ vfs_caches_init_early(); mem_init(); kmem_cache_init(); + cart_init(); numa_policy_init(); if (late_time_init) late_time_init(); diff -Naur /home/rni/linux-2.6.12-rc5/mm/cart.c linux-cart.v3/mm/cart.c --- /home/rni/linux-2.6.12-rc5/mm/cart.c 1969-12-31 19:00:00.000000000 -0500 +++ linux-cart.v3/mm/cart.c 2005-08-16 14:06:12.000000000 -0400 @@ -0,0 +1,282 @@ +/* Thisfile contains the crux of the CART page replacement algorithm. This implementation however changes a few things form the classic CART scheme. This implementation splits the original active_list of the Linux implementation into two lists, namely active_list and active_longterm. The 'active' pages exist on these two lists. The active_list hopes to capture short term usage, while the active_longterm list hopes to capture long term usage. Whenever a page's state needs to be updated, the update_cart_params() function is called. The refill_incative_zone() function causes the replace() function to be evoked, resulting in the removal of pages from the active lists. Hence, which pages are deemed inactive is determined by the CART algorithm. +For further details, please refer to the CART paper here - http://www.almaden.ibm.com/cs/people/dmodha/clockfast.pdf */ + +#include +#include +#include +#include +#include +#include +#include +#include + +kmem_cache_t *evicted_node_cache; +mempool_t *evicted_node_pool; + +/* Called from init/main.c to initialize the cart parameters */ +void cart_init() +{ + pg_data_t *pgdat; + struct zone *zone; + int i; + + pgdat = pgdat_list; + + do { + for (i=0;inode_zones[i]; + + spin_lock_init(&zone->lru_lock); + INIT_LIST_HEAD(&zone->active_list); + INIT_LIST_HEAD(&zone->active_longterm); + INIT_LIST_HEAD(&zone->inactive_list); + INIT_LIST_HEAD(&zone->evicted_active); + INIT_LIST_HEAD(&zone->evicted_longterm); + + hashtable_init(&zone->evicted_active_hashtable); + hashtable_init(&zone->evicted_longterm_hashtable); + + zone->nr_active = zone->nr_active_longterm = zone->nr_inactive = zone->nr_evicted_active = zone->nr_evicted_longterm = 0; + + zone->p = zone->q = zone->nr_longterm = zone->nr_shortterm = 0; + } + } while ((pgdat = pgdat->pgdat_next)); + + /* Create a slab for non resident nodes */ + evicted_node_cache = kmem_cache_create("EvictedPool", sizeof(struct non_res_list_node), 0, 0, NULL, NULL); + + if (!evicted_node_cache) + panic("Could not allocate evicted node cache!\n"); + + /* Create a mempool of preallocated objects */ + evicted_node_pool = mempool_create(MIN_POOL, mempool_alloc_slab, mempool_free_slab, evicted_node_cache); + + if (!evicted_node_pool) + panic("Could not allocate evicted node_pool!\n"); + +} + +/* Add a node to the evicted list specified */ +void add_to_evicted(struct hashtable *h, struct list_head *l, struct page *page) +{ + struct non_res_list_node *node; + node = mempool_alloc(evicted_node_pool, GFP_ATOMIC); + + if (!node) + /* This is usually bad news :) */ + printk (KERN_EMERG "Couldn't get a non_res_node!\n"); + return; + + node->hashval = mk_hash_page(page); + + list_add(&node->list, l); + add_to_hashtable(h, node); +} + +/* Delete a node form the tail of an evicted list */ +void del_from_evicted_tail(struct list_head *head) +{ + struct list_head *list; + struct non_res_list_node *node; + if(list_empty(head)) + return; + + list = head->prev; + node = list_entry(list, struct non_res_list_node, list); + list_del(list); + list_del(&node->hash); + + mempool_free(node, evicted_node_pool); +} + +/* Remove the tail node of the evicted active list */ +void del_from_evicted_active_tail(struct zone *zone) +{ + del_from_evicted_tail(&zone->evicted_active); +} + +/* Remove the tail node of the evicted longterm list */ +void del_from_evicted_longterm_tail(struct zone *zone) +{ + del_from_evicted_tail(&zone->evicted_longterm); +} + +/* Add a node to the evicted active list */ +void add_to_evicted_active(struct zone *zone, struct page *page) +{ + add_to_evicted(&zone->evicted_active_hashtable, &zone->evicted_active, page); + ++zone->nr_evicted_active; +} + +/* Add a node to the evicted longterm list */ +void add_to_evicted_longterm(struct zone *zone, struct page *page) +{ + add_to_evicted(&zone->evicted_longterm_hashtable, &zone->evicted_longterm, page); + ++zone->nr_evicted_longterm; +} + +/* Search for a node in the evicted active list */ +unsigned long find_in_evicted_active(struct page *page) +{ + struct zone *zone; + zone = page_zone(page); + return find_in_hashtable(&zone->evicted_active_hashtable, page); +} + +/* Search for a node in the evicted longterm list */ +unsigned long find_in_evicted_longterm(struct page *page) +{ + struct zone *zone; + zone = page_zone(page); + return find_in_hashtable(&zone->evicted_longterm_hashtable, page); +} + +/* Look to see whether a node is in any evicted list */ +unsigned long find_in_evicted_list(struct page *page) +{ + if (find_in_evicted_active(page)) + return EVICTED_ACTIVE; + if (find_in_evicted_longterm(page)) + return EVICTED_LONGTERM; + + return 0; +} + +/* The heart of the CART update function. This function is responsible for the movement of pages across the lists */ +void update_cart_params(struct page *page) +{ + unsigned long location; + unsigned long evicted_active; + unsigned evicted_longterm; + struct zone *zone; + + zone = page_zone(page); + + location = find_in_evicted_list(page); + evicted_active = EvictedActive(location); + evicted_longterm = EvictedLongterm(location); + + if (evicted_active) { + zone->p = min(zone->p + max(zone->nr_shortterm/zone->nr_evicted_active, 1), zone->pages_high); + + ++zone->nr_longterm; + SetLongTerm(page); + ClearPageReferenced(page); + } + else if (evicted_longterm) { + zone->p = max(zone->p - max(1, zone->nr_longterm/zone->nr_evicted_longterm), 0); + ++zone->nr_longterm; + ClearPageReferenced(page); + + if (zone->nr_active_longterm + zone->nr_active + zone->nr_evicted_longterm - zone->nr_shortterm >=zone->pages_high) { + zone->q = min(zone->q + 1, 2*zone->pages_high - zone->nr_active); + } + } + else { + ++zone->nr_shortterm; + ClearLongTerm(page); + } + + add_page_to_active_list(zone, page); +} + +/* The replace function. This function serches the active and longterm lists and looks for a candidate for replacement. This function selects the candidate and returns the corresponding structpage or returns NULL in case no page can be freed. The *where argument is used to indicate the parent list of the page so that, in case it cannot be written back, it can be placed back on the correct list */ +struct page *replace(struct zone *zone, int *where) +{ + struct list_head *list; + struct page *page = NULL; + int referenced = 0; + unsigned long location; + int debug_count=0; + + list = &zone->active_longterm; + list = list->next; + while (list !=&zone->active_longterm) { + page = list_entry(list, struct page, lru); + + if (!PageReferenced(page)) + break; + + ClearPageReferenced(page); + del_page_from_active_longterm(zone, page); + add_page_to_active_list_tail(zone, page); + + if ((zone->nr_active_longterm + zone->nr_active + zone->nr_evicted_longterm - zone->nr_shortterm) >= zone->pages_high) + zone->q = min(zone->q + 1, 2*zone->pages_high - zone->nr_active); + + list = &zone->active_longterm; + list = list->next; + debug_count++; + } + + debug_count=0; + list = &zone->active_list; + list = list->next; + + while (list != &zone->active_list) { + page = list_entry(list, struct page, lru); + referenced = PageReferenced(page); + + if (!PageLongTerm(page) && !referenced) + break; + + ClearPageReferenced(page); + if (referenced) { + del_page_from_active_list(zone, page); + add_page_to_active_list_tail(zone, page); + + if (zone->nr_active >= min(zone->p+1, zone->nr_evicted_active) && !PageLongTerm(page)) { + SetLongTerm(page); + --zone->nr_shortterm; + ++zone->nr_longterm; + } + } + else { + del_page_from_active_list(zone, page); + add_page_to_active_longterm_tail(zone, page); + + zone->q = max(zone->q-1, zone->pages_high - zone->nr_active); + } + + list = &zone->active_list; + list = list->next; + debug_count++; + } + + page = NULL; + + if (zone->nr_active > max(1, zone->p)) { + if (!list_empty(&zone->active_list)) { + page = list_entry(zone->active_list.next, struct page, lru); + del_page_from_active_list(zone, page); + add_to_evicted_active(zone, page); + *where = ACTIVE; + } + } + else { + if (!list_empty(&zone->active_longterm)) { + page = list_entry(zone->active_longterm.next, struct page, lru); + del_page_from_active_longterm(zone, page); + --zone->nr_longterm; + add_to_evicted_longterm(zone, page); + *where = ACTIVE_LONGTERM; + } + } + + if (!page) + return NULL; + + if (*where == 0) + BUG(); + + location = find_in_evicted_list(page); + + if (!location && (zone->nr_evicted_active + zone->nr_evicted_longterm == zone->pages_high+1) && ((zone->nr_evicted_active > max(0, zone->q) ||zone->nr_evicted_longterm == 0))) { + del_from_evicted_active_tail(zone); + } + else if (!location && (zone->nr_evicted_active + zone->nr_evicted_longterm == zone->pages_high + 1)) + del_from_evicted_longterm_tail(zone); + + return page; +} + diff -Naur /home/rni/linux-2.6.12-rc5/mm/evicted_hash.c linux-cart.v3/mm/evicted_hash.c --- /home/rni/linux-2.6.12-rc5/mm/evicted_hash.c 1969-12-31 19:00:00.000000000 -0500 +++ linux-cart.v3/mm/evicted_hash.c 2005-08-16 14:05:29.000000000 -0400 @@ -0,0 +1,73 @@ +/* This file contains the functions required to manage the evicted list hash table. The evicted nodes are maintained as a couple of lists, but the hash table is used for speedy lookup */ + +#include +#include +#include + +/* Initialize the hashtable */ +void hashtable_init(struct hashtable *h) +{ + int i; + + for (i=0;ibuckets[i]); + } +} + +/* Get the inode number of a page if it is file backed, return 0 if it is anonymous */ +unsigned long get_inode_num(void *addr) +{ + struct address_space *p; + + if (!addr) + return 0; + + /* If lower bit is set, then it is a anon_vma object */ + if (((unsigned long)addr) & 0x1) + return 0; + + p = (struct address_space *)addr; + + return p->host->i_ino; +} + +/* The hashing function... a better one is needed */ +static inline unsigned long mk_hash_page(struct page *page) +{ + return ((unsigned long)page->mapping ^ page->index) ^ get_inode_num(page->mapping); +} + +/* Hashing for non resident nodes */ +static inline unsigned long mk_hash_non_res(struct non_res_list_node *node) +{ + return (node->hashval)%HASHTABLE_SIZE; +} + +/* Search in the hash table */ +unsigned long find_in_hashtable(struct hashtable *h, struct page *page) +{ + unsigned long index; + struct non_res_list_node *node; + struct list_head *list; + unsigned long hashval = mk_hash_page(page); + + index = hashval%HASHTABLE_SIZE; + + list_for_each_entry(node, &h->buckets[index], hash) { + if (node->hashval == hashval) + return 1; + } + + return 0; +} + +/* Add a node to the hash table */ +void add_to_hashtable(struct hashtable *h, struct non_res_list_node *node) +{ + unsigned long index; + + index = mk_hash_non_res(node); + + list_add(&node->hash, &h->buckets[index]); +} + diff -Naur /home/rni/linux-2.6.12-rc5/mm/filemap.c linux-cart.v3/mm/filemap.c --- /home/rni/linux-2.6.12-rc5/mm/filemap.c 2005-05-24 23:31:20.000000000 -0400 +++ linux-cart.v3/mm/filemap.c 2005-08-16 13:59:51.000000000 -0400 @@ -107,7 +107,6 @@ void __remove_from_page_cache(struct page *page) { struct address_space *mapping = page->mapping; - radix_tree_delete(&mapping->page_tree, page->index); page->mapping = NULL; mapping->nrpages--; diff -Naur /home/rni/linux-2.6.12-rc5/mm/Makefile linux-cart.v3/mm/Makefile --- /home/rni/linux-2.6.12-rc5/mm/Makefile 2005-05-24 23:31:20.000000000 -0400 +++ linux-cart.v3/mm/Makefile 2005-08-16 13:59:51.000000000 -0400 @@ -5,7 +5,7 @@ mmu-y := nommu.o mmu-$(CONFIG_MMU) := fremap.o highmem.o madvise.o memory.o mincore.o \ mlock.o mmap.o mprotect.o mremap.o msync.o rmap.o \ - vmalloc.o + vmalloc.o cart.o evicted_hash.o obj-y := bootmem.o filemap.o mempool.o oom_kill.o fadvise.o \ page_alloc.o page-writeback.o pdflush.o \ diff -Naur /home/rni/linux-2.6.12-rc5/mm/swap.c linux-cart.v3/mm/swap.c --- /home/rni/linux-2.6.12-rc5/mm/swap.c 2005-05-24 23:31:20.000000000 -0400 +++ linux-cart.v3/mm/swap.c 2005-08-16 13:59:51.000000000 -0400 @@ -30,6 +30,7 @@ #include #include #include +#include /* How many pages do we try to swap or page in/out together? */ int page_cluster; @@ -107,7 +108,7 @@ if (PageLRU(page) && !PageActive(page)) { del_page_from_inactive_list(zone, page); SetPageActive(page); - add_page_to_active_list(zone, page); + update_cart_params(page); inc_page_state(pgactivate); } spin_unlock_irq(&zone->lru_lock); @@ -124,7 +125,6 @@ { if (!PageActive(page) && PageReferenced(page) && PageLRU(page)) { activate_page(page); - ClearPageReferenced(page); } else if (!PageReferenced(page)) { SetPageReferenced(page); } diff -Naur /home/rni/linux-2.6.12-rc5/mm/vmscan.c linux-cart.v3/mm/vmscan.c --- /home/rni/linux-2.6.12-rc5/mm/vmscan.c 2005-05-24 23:31:20.000000000 -0400 +++ linux-cart.v3/mm/vmscan.c 2005-08-16 13:59:51.000000000 -0400 @@ -38,6 +38,7 @@ #include #include +#include /* possible outcome of pageout() */ typedef enum { @@ -545,6 +546,50 @@ return reclaimed; } +/* This gets a page from the active_list and active_longterm lists in order to add to the incative list */ +static int get_from_active_lists(int nr_to_scan, struct zone *zone, struct list_head *dst, int *scanned) +{ + int nr_taken = 0; + struct page *page; + int scan = 0; + int location; + + while (scan++ < nr_to_scan) { + location = 0; + page = replace(zone, &location); + + if (!page) + break; + + if (!TestClearPageLRU(page)) { + BUG(); + } + + if (!location) + BUG(); + + if (get_page_testone(page)) { + /* + * It is being freed elsewhere + */ + __put_page(page); + SetPageLRU(page); + + if (Active(location)) + add_page_to_active_list_tail(zone, page); + else + add_page_to_active_longterm_tail(zone, page); + continue; + } else { + list_add(&page->lru, dst); + nr_taken++; + } + } + + *scanned = scan; + return nr_taken; +} + /* * zone->lru_lock is heavily contended. Some of the functions that * shrink the lists perform better by taking out a batch of pages @@ -695,10 +740,10 @@ lru_add_drain(); spin_lock_irq(&zone->lru_lock); - pgmoved = isolate_lru_pages(nr_pages, &zone->active_list, + pgmoved = get_from_active_lists(nr_pages, zone, &l_hold, &pgscanned); zone->pages_scanned += pgscanned; - zone->nr_active -= pgmoved; +// zone->nr_active -= pgmoved; spin_unlock_irq(&zone->lru_lock); /*