diff --git a/kernel/arch/arm/configs/uckernel_defconfig b/kernel/arch/arm/configs/uckernel_defconfig index c157fedf..c3f5dae9 100644 --- a/kernel/arch/arm/configs/uckernel_defconfig +++ b/kernel/arch/arm/configs/uckernel_defconfig @@ -1990,7 +1990,14 @@ CONFIG_ANDROID_LOW_MEMORY_KILLER=y # RAR Register Driver # # CONFIG_IIO is not set +CONFIG_XVMALLOC=y CONFIG_ZRAM=y +CONFIG_ZRAM_NUM_DEVICES=1 +CONFIG_ZRAM_DEFAULT_PERCENTAGE=25 +# CONFIG_ZRAM_DEBUG is not set +CONFIG_ZRAM_LZO=y +# CONFIG_ZRAM_SNAPPY is not set +CONFIG_ZRAM_DEFAULT_DISKSIZE=100663296 # CONFIG_BATMAN_ADV is not set # CONFIG_FB_SM7XX is not set diff --git a/kernel/drivers/staging/Makefile b/kernel/drivers/staging/Makefile index 6cbcae32..5f26704f 100755 --- a/kernel/drivers/staging/Makefile +++ b/kernel/drivers/staging/Makefile @@ -40,7 +40,8 @@ obj-$(CONFIG_RAR_REGISTER) += rar_register/ obj-$(CONFIG_MRST_RAR_HANDLER) += memrar/ obj-$(CONFIG_DX_SEP) += sep/ obj-$(CONFIG_IIO) += iio/ -obj-$(CONFIG_ZRAM) += zram/ +obj-$(CONFIG_ZRAM) += zram/ +obj-$(CONFIG_XVMALLOC) += zram/ obj-$(CONFIG_WLAGS49_H2) += wlags49_h2/ obj-$(CONFIG_WLAGS49_H25) += wlags49_h25/ obj-$(CONFIG_BATMAN_ADV) += batman-adv/ diff --git a/kernel/drivers/staging/zram/Kconfig b/kernel/drivers/staging/zram/Kconfig index da079f8d..16a7d454 100644 --- a/kernel/drivers/staging/zram/Kconfig +++ b/kernel/drivers/staging/zram/Kconfig @@ -1,8 +1,11 @@ +config XVMALLOC + bool + default n + config ZRAM tristate "Compressed RAM block device support" - depends on BLOCK - select LZO_COMPRESS - select LZO_DECOMPRESS + depends on BLOCK && SYSFS + select XVMALLOC default n help Creates virtual block devices called /dev/zramX (X = 0, 1, ...). @@ -15,3 +18,54 @@ config ZRAM See zram.txt for more information. Project home: http://compcache.googlecode.com/ + +config ZRAM_NUM_DEVICES + int "Default number of zram devices" + depends on ZRAM + range 1 32 + default 1 + help + Select default number of zram devices. You can override this value + using 'num_devices' module parameter. + +config ZRAM_DEFAULT_PERCENTAGE + int "Default number of zram percentage" + depends on ZRAM + range 10 80 + default 25 + help + Select default zram disk size: percentage of total RAM + +config ZRAM_DEBUG + bool "Compressed RAM block device debug support" + depends on ZRAM + default n + help + This option adds additional debugging code to the compressed + RAM block device driver. + +choice ZRAM_COMPRESS + prompt "compression method" + depends on ZRAM + default ZRAM_LZO + help + Select the compression method used by zram. + LZO is the default. Snappy compresses a bit worse (around ~2%) but + much (~2x) faster, at least on x86-64. +config ZRAM_LZO + bool "LZO compression" + select LZO_COMPRESS + select LZO_DECOMPRESS +config ZRAM_SNAPPY + bool "Snappy compression" + depends on SNAPPY_COMPRESS + depends on SNAPPY_DECOMPRESS +endchoice + +config ZRAM_DEFAULT_DISKSIZE + int "Default size of zram in bytes" + depends on ZRAM + default 100663296 + help + Set default zram disk size (default ~ 96MB) + diff --git a/kernel/drivers/staging/zram/Makefile b/kernel/drivers/staging/zram/Makefile index b1709c57..2a6d3213 100644 --- a/kernel/drivers/staging/zram/Makefile +++ b/kernel/drivers/staging/zram/Makefile @@ -1,3 +1,4 @@ -zram-y := zram_drv.o zram_sysfs.o xvmalloc.o +zram-y := zram_drv.o zram_sysfs.o obj-$(CONFIG_ZRAM) += zram.o +obj-$(CONFIG_XVMALLOC) += xvmalloc.o \ No newline at end of file diff --git a/kernel/drivers/staging/zram/xvmalloc.c b/kernel/drivers/staging/zram/xvmalloc.c index 3fdbb8ad..1f9c5082 100644 --- a/kernel/drivers/staging/zram/xvmalloc.c +++ b/kernel/drivers/staging/zram/xvmalloc.c @@ -10,6 +10,12 @@ * Released under the terms of GNU General Public License Version 2.0 */ +#ifdef CONFIG_ZRAM_DEBUG +#define DEBUG +#endif + +#include +#include #include #include #include @@ -46,7 +52,7 @@ static void clear_flag(struct block_header *block, enum blockflags flag) } /* - * Given pair, provide a derefrencable pointer. + * Given pair, provide a dereferencable pointer. * This is called from xv_malloc/xv_free path, so it * needs to be fast. */ @@ -187,7 +193,7 @@ static void insert_block(struct xv_pool *pool, struct page *page, u32 offset, slindex = get_index_for_insert(block->size); flindex = slindex / BITS_PER_LONG; - block->link.prev_page = 0; + block->link.prev_page = NULL; block->link.prev_offset = 0; block->link.next_page = pool->freelist[slindex].page; block->link.next_offset = pool->freelist[slindex].offset; @@ -200,61 +206,23 @@ static void insert_block(struct xv_pool *pool, struct page *page, u32 offset, nextblock->link.prev_page = page; nextblock->link.prev_offset = offset; put_ptr_atomic(nextblock, KM_USER1); + /* If there was a next page then the free bits are set. */ + return; } __set_bit(slindex % BITS_PER_LONG, &pool->slbitmap[flindex]); __set_bit(flindex, &pool->flbitmap); } -/* - * Remove block from head of freelist. Index 'slindex' identifies the freelist. - */ -static void remove_block_head(struct xv_pool *pool, - struct block_header *block, u32 slindex) -{ - struct block_header *tmpblock; - u32 flindex = slindex / BITS_PER_LONG; - - pool->freelist[slindex].page = block->link.next_page; - pool->freelist[slindex].offset = block->link.next_offset; - block->link.prev_page = 0; - block->link.prev_offset = 0; - - if (!pool->freelist[slindex].page) { - __clear_bit(slindex % BITS_PER_LONG, &pool->slbitmap[flindex]); - if (!pool->slbitmap[flindex]) - __clear_bit(flindex, &pool->flbitmap); - } else { - /* - * DEBUG ONLY: We need not reinitialize freelist head previous - * pointer to 0 - we never depend on its value. But just for - * sanity, lets do it. - */ - tmpblock = get_ptr_atomic(pool->freelist[slindex].page, - pool->freelist[slindex].offset, KM_USER1); - tmpblock->link.prev_page = 0; - tmpblock->link.prev_offset = 0; - put_ptr_atomic(tmpblock, KM_USER1); - } -} - /* * Remove block from freelist. Index 'slindex' identifies the freelist. */ static void remove_block(struct xv_pool *pool, struct page *page, u32 offset, struct block_header *block, u32 slindex) { - u32 flindex; + u32 flindex = slindex / BITS_PER_LONG; struct block_header *tmpblock; - if (pool->freelist[slindex].page == page - && pool->freelist[slindex].offset == offset) { - remove_block_head(pool, block, slindex); - return; - } - - flindex = slindex / BITS_PER_LONG; - if (block->link.prev_page) { tmpblock = get_ptr_atomic(block->link.prev_page, block->link.prev_offset, KM_USER1); @@ -270,6 +238,35 @@ static void remove_block(struct xv_pool *pool, struct page *page, u32 offset, tmpblock->link.prev_offset = block->link.prev_offset; put_ptr_atomic(tmpblock, KM_USER1); } + + /* Is this block is at the head of the freelist? */ + if (pool->freelist[slindex].page == page + && pool->freelist[slindex].offset == offset) { + + pool->freelist[slindex].page = block->link.next_page; + pool->freelist[slindex].offset = block->link.next_offset; + + if (pool->freelist[slindex].page) { + struct block_header *tmpblock; + tmpblock = get_ptr_atomic(pool->freelist[slindex].page, + pool->freelist[slindex].offset, + KM_USER1); + tmpblock->link.prev_page = NULL; + tmpblock->link.prev_offset = 0; + put_ptr_atomic(tmpblock, KM_USER1); + } else { + /* This freelist bucket is empty */ + __clear_bit(slindex % BITS_PER_LONG, + &pool->slbitmap[flindex]); + if (!pool->slbitmap[flindex]) + __clear_bit(flindex, &pool->flbitmap); + } + } + + block->link.prev_page = NULL; + block->link.prev_offset = 0; + block->link.next_page = NULL; + block->link.next_offset = 0; } /* @@ -320,11 +317,13 @@ struct xv_pool *xv_create_pool(void) return pool; } +EXPORT_SYMBOL_GPL(xv_create_pool); void xv_destroy_pool(struct xv_pool *pool) { kfree(pool); } +EXPORT_SYMBOL_GPL(xv_destroy_pool); /** * xv_malloc - Allocate block of given size from pool. @@ -378,7 +377,7 @@ int xv_malloc(struct xv_pool *pool, u32 size, struct page **page, block = get_ptr_atomic(*page, *offset, KM_USER0); - remove_block_head(pool, block, index); + remove_block(pool, *page, *offset, block, index); /* Split the block if required */ tmpoffset = *offset + size + XV_ALIGN; @@ -413,6 +412,7 @@ int xv_malloc(struct xv_pool *pool, u32 size, struct page **page, return 0; } +EXPORT_SYMBOL_GPL(xv_malloc); /* * Free block identified with @@ -489,6 +489,7 @@ void xv_free(struct xv_pool *pool, struct page *page, u32 offset) put_ptr_atomic(page_start, KM_USER0); spin_unlock(&pool->lock); } +EXPORT_SYMBOL_GPL(xv_free); u32 xv_get_object_size(void *obj) { @@ -497,6 +498,7 @@ u32 xv_get_object_size(void *obj) blk = (struct block_header *)((char *)(obj) - XV_ALIGN); return blk->size; } +EXPORT_SYMBOL_GPL(xv_get_object_size); /* * Returns total memory used by allocator (userdata + metadata) @@ -505,3 +507,4 @@ u64 xv_get_total_size_bytes(struct xv_pool *pool) { return pool->total_pages << PAGE_SHIFT; } +EXPORT_SYMBOL_GPL(xv_get_total_size_bytes); diff --git a/kernel/drivers/staging/zram/xvmalloc_int.h b/kernel/drivers/staging/zram/xvmalloc_int.h index e23ed5c8..b5f1f7fe 100644 --- a/kernel/drivers/staging/zram/xvmalloc_int.h +++ b/kernel/drivers/staging/zram/xvmalloc_int.h @@ -19,7 +19,11 @@ /* User configurable params */ /* Must be power of two */ +#ifdef CONFIG_64BIT +#define XV_ALIGN_SHIFT 3 +#else #define XV_ALIGN_SHIFT 2 +#endif #define XV_ALIGN (1 << XV_ALIGN_SHIFT) #define XV_ALIGN_MASK (XV_ALIGN - 1) @@ -27,8 +31,16 @@ #define XV_MIN_ALLOC_SIZE 32 #define XV_MAX_ALLOC_SIZE (PAGE_SIZE - XV_ALIGN) -/* Free lists are separated by FL_DELTA bytes */ -#define FL_DELTA_SHIFT 3 +/* + * Free lists are separated by FL_DELTA bytes + * This value is 3 for 4k pages and 4 for 64k pages, for any + * other page size, a conservative (PAGE_SHIFT - 9) is used. + */ +#if PAGE_SHIFT == 16 +#define FL_DELTA_SHIFT 4 +#else +#define FL_DELTA_SHIFT (PAGE_SHIFT - 9) +#endif #define FL_DELTA (1 << FL_DELTA_SHIFT) #define FL_DELTA_MASK (FL_DELTA - 1) #define NUM_FREE_LISTS ((XV_MAX_ALLOC_SIZE - XV_MIN_ALLOC_SIZE) \ @@ -75,12 +87,9 @@ struct block_header { struct xv_pool { ulong flbitmap; ulong slbitmap[MAX_FLI]; - spinlock_t lock; - + u64 total_pages; /* stats */ struct freelist_entry freelist[NUM_FREE_LISTS]; - - /* stats */ - u64 total_pages; + spinlock_t lock; }; #endif diff --git a/kernel/drivers/staging/zram/zram_drv.c b/kernel/drivers/staging/zram/zram_drv.c index 080e85fe..c55fc11f 100644 --- a/kernel/drivers/staging/zram/zram_drv.c +++ b/kernel/drivers/staging/zram/zram_drv.c @@ -15,6 +15,10 @@ #define KMSG_COMPONENT "zram" #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt +#ifdef CONFIG_ZRAM_DEBUG +#define DEBUG +#endif + #include #include #include @@ -25,18 +29,62 @@ #include #include #include -#include #include #include #include "zram_drv.h" +#if defined(CONFIG_ZRAM_LZO) +#include +#define WMSIZE LZO1X_MEM_COMPRESS +#define COMPRESS(s, sl, d, dl, wm) \ + lzo1x_1_compress(s, sl, d, dl, wm) +#define DECOMPRESS(s, sl, d, dl) \ + lzo1x_decompress_safe(s, sl, d, dl) +#elif defined(CONFIG_ZRAM_SNAPPY) +#include "../snappy/csnappy.h" /* if built in drivers/staging */ +#define WMSIZE_ORDER ((PAGE_SHIFT > 14) ? (15) : (PAGE_SHIFT+1)) +#define WMSIZE (1 << WMSIZE_ORDER) +static int +snappy_compress_( + const unsigned char *src, + size_t src_len, + unsigned char *dst, + size_t *dst_len, + void *workmem) +{ + const unsigned char *end = csnappy_compress_fragment( + src, (uint32_t)src_len, dst, workmem, WMSIZE_ORDER); + *dst_len = end - dst; + return 0; +} +static int +snappy_decompress_( + const unsigned char *src, + size_t src_len, + unsigned char *dst, + size_t *dst_len) +{ + uint32_t dst_len_ = (uint32_t)*dst_len; + int ret = csnappy_decompress_noheader(src, src_len, dst, &dst_len_); + *dst_len = (size_t)dst_len_; + return ret; +} +#define COMPRESS(s, sl, d, dl, wm) \ + snappy_compress_(s, sl, d, dl, wm) +#define DECOMPRESS(s, sl, d, dl) \ + snappy_decompress_(s, sl, d, dl) +#else +#error either CONFIG_ZRAM_LZO or CONFIG_ZRAM_SNAPPY must be defined +#endif + + /* Globals */ static int zram_major; -struct zram *devices; +struct zram *zram_devices; /* Module params (documentation at end) */ -unsigned int num_devices; +unsigned int zram_num_devices; static void zram_stat_inc(u32 *v) { @@ -100,33 +148,19 @@ static int page_zero_filled(void *ptr) return 1; } -static void zram_set_disksize(struct zram *zram, size_t totalram_bytes) +static u64 zram_default_disksize_bytes(void) { - if (!zram->disksize) { - pr_info( - "disk size not provided. You can use disksize_kb module " - "param to specify size.\nUsing default: (%u%% of RAM).\n", - default_disksize_perc_ram - ); - zram->disksize = default_disksize_perc_ram * - (totalram_bytes / 100); - } - - if (zram->disksize > 2 * (totalram_bytes)) { - pr_info( - "There is little point creating a zram of greater than " - "twice the size of memory since we expect a 2:1 compression " - "ratio. Note that zram uses about 0.1%% of the size of " - "the disk when not in use so a huge zram is " - "wasteful.\n" - "\tMemory Size: %zu kB\n" - "\tSize you selected: %llu kB\n" - "Continuing anyway ...\n", - totalram_bytes >> 10, zram->disksize - ); - } +#if 0 + return ((totalram_pages << PAGE_SHIFT) * + default_disksize_perc_ram / 100) & PAGE_MASK; +#endif + return CONFIG_ZRAM_DEFAULT_DISKSIZE; +} - zram->disksize &= PAGE_MASK; +static void zram_set_disksize(struct zram *zram, u64 size_bytes) +{ + zram->disksize = size_bytes; + set_capacity(zram->disk, size_bytes >> SECTOR_SHIFT); } static void zram_free_page(struct zram *zram, size_t index) @@ -173,256 +207,376 @@ static void zram_free_page(struct zram *zram, size_t index) zram->table[index].offset = 0; } -static void handle_zero_page(struct page *page) +static void handle_zero_page(struct bio_vec *bvec) { + struct page *page = bvec->bv_page; void *user_mem; user_mem = kmap_atomic(page, KM_USER0); - memset(user_mem, 0, PAGE_SIZE); + memset(user_mem + bvec->bv_offset, 0, bvec->bv_len); kunmap_atomic(user_mem, KM_USER0); flush_dcache_page(page); } -static void handle_uncompressed_page(struct zram *zram, - struct page *page, u32 index) +static void handle_uncompressed_page(struct zram *zram, struct bio_vec *bvec, + u32 index, int offset) { + struct page *page = bvec->bv_page; unsigned char *user_mem, *cmem; user_mem = kmap_atomic(page, KM_USER0); - cmem = kmap_atomic(zram->table[index].page, KM_USER1) + - zram->table[index].offset; + cmem = kmap_atomic(zram->table[index].page, KM_USER1); - memcpy(user_mem, cmem, PAGE_SIZE); - kunmap_atomic(user_mem, KM_USER0); + memcpy(user_mem + bvec->bv_offset, cmem + offset, bvec->bv_len); kunmap_atomic(cmem, KM_USER1); + kunmap_atomic(user_mem, KM_USER0); flush_dcache_page(page); } -static int zram_read(struct zram *zram, struct bio *bio) +static inline int is_partial_io(struct bio_vec *bvec) { + return bvec->bv_len != PAGE_SIZE; +} - int i; - u32 index; - struct bio_vec *bvec; +static int zram_bvec_read(struct zram *zram, struct bio_vec *bvec, + u32 index, int offset, struct bio *bio) +{ + int ret; + size_t clen; + struct page *page; + struct zobj_header *zheader; + unsigned char *user_mem, *cmem, *uncmem = NULL; + + page = bvec->bv_page; - if (unlikely(!zram->init_done)) { - set_bit(BIO_UPTODATE, &bio->bi_flags); - bio_endio(bio, 0); + if (zram_test_flag(zram, index, ZRAM_ZERO)) { + handle_zero_page(bvec); return 0; } - zram_stat64_inc(zram, &zram->stats.num_reads); - index = bio->bi_sector >> SECTORS_PER_PAGE_SHIFT; - - bio_for_each_segment(bvec, bio, i) { - int ret; - size_t clen; - struct page *page; - struct zobj_header *zheader; - unsigned char *user_mem, *cmem; - - page = bvec->bv_page; - - if (zram_test_flag(zram, index, ZRAM_ZERO)) { - handle_zero_page(page); - index++; - continue; - } + /* Requested page is not present in compressed area */ + if (unlikely(!zram->table[index].page)) { + pr_debug("Read before write: sector=%lu, size=%u", + (ulong)(bio->bi_sector), bio->bi_size); + handle_zero_page(bvec); + return 0; + } - /* Requested page is not present in compressed area */ - if (unlikely(!zram->table[index].page)) { - pr_debug("Read before write: sector=%lu, size=%u", - (ulong)(bio->bi_sector), bio->bi_size); - /* Do nothing */ - index++; - continue; - } + /* Page is stored uncompressed since it's incompressible */ + if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) { + handle_uncompressed_page(zram, bvec, index, offset); + return 0; + } - /* Page is stored uncompressed since it's incompressible */ - if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) { - handle_uncompressed_page(zram, page, index); - index++; - continue; + if (is_partial_io(bvec)) { + /* Use a temporary buffer to decompress the page */ + uncmem = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!uncmem) { + pr_info("Error allocating temp memory!\n"); + return -ENOMEM; } + } - user_mem = kmap_atomic(page, KM_USER0); - clen = PAGE_SIZE; + user_mem = kmap_atomic(page, KM_USER0); + if (!is_partial_io(bvec)) + uncmem = user_mem; + clen = PAGE_SIZE; - cmem = kmap_atomic(zram->table[index].page, KM_USER1) + - zram->table[index].offset; + cmem = kmap_atomic(zram->table[index].page, KM_USER1) + + zram->table[index].offset; - ret = lzo1x_decompress_safe( + ret = DECOMPRESS( cmem + sizeof(*zheader), xv_get_object_size(cmem) - sizeof(*zheader), - user_mem, &clen); + uncmem, &clen); - kunmap_atomic(user_mem, KM_USER0); - kunmap_atomic(cmem, KM_USER1); + if (is_partial_io(bvec)) { + memcpy(user_mem + bvec->bv_offset, uncmem + offset, + bvec->bv_len); + kfree(uncmem); + } - /* Should NEVER happen. Return bio error if it does. */ - if (unlikely(ret != LZO_E_OK)) { - pr_err("Decompression failed! err=%d, page=%u\n", - ret, index); - zram_stat64_inc(zram, &zram->stats.failed_reads); - goto out; - } + kunmap_atomic(cmem, KM_USER1); + kunmap_atomic(user_mem, KM_USER0); - flush_dcache_page(page); - index++; + /* Should NEVER happen. Return bio error if it does. */ + if (unlikely(ret)) { + pr_err("Decompression failed! err=%d, page=%u\n", ret, index); + zram_stat64_inc(zram, &zram->stats.failed_reads); + return ret; } - set_bit(BIO_UPTODATE, &bio->bi_flags); - bio_endio(bio, 0); - return 0; + flush_dcache_page(page); -out: - bio_io_error(bio); return 0; } -static int zram_write(struct zram *zram, struct bio *bio) +static int zram_read_before_write(struct zram *zram, char *mem, u32 index) { - int i, ret; - u32 index; - struct bio_vec *bvec; + int ret; + size_t clen = PAGE_SIZE; + struct zobj_header *zheader; + unsigned char *cmem; - if (unlikely(!zram->init_done)) { - ret = zram_init_device(zram); - if (ret) - goto out; + if (zram_test_flag(zram, index, ZRAM_ZERO) || + !zram->table[index].page) { + memset(mem, 0, PAGE_SIZE); + return 0; } - zram_stat64_inc(zram, &zram->stats.num_writes); - index = bio->bi_sector >> SECTORS_PER_PAGE_SHIFT; + cmem = kmap_atomic(zram->table[index].page, KM_USER0) + + zram->table[index].offset; - bio_for_each_segment(bvec, bio, i) { - u32 offset; - size_t clen; - struct zobj_header *zheader; - struct page *page, *page_store; - unsigned char *user_mem, *cmem, *src; + /* Page is stored uncompressed since it's incompressible */ + if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) { + memcpy(mem, cmem, PAGE_SIZE); + kunmap_atomic(cmem, KM_USER0); + return 0; + } - page = bvec->bv_page; - src = zram->compress_buffer; + ret = DECOMPRESS(cmem + sizeof(*zheader), + xv_get_object_size(cmem) - sizeof(*zheader), + mem, &clen); + kunmap_atomic(cmem, KM_USER0); + + /* Should NEVER happen. Return bio error if it does. */ + if (unlikely(ret)) { + pr_err("Decompression failed! err=%d, page=%u\n", ret, index); + zram_stat64_inc(zram, &zram->stats.failed_reads); + return ret; + } + + return 0; +} + +static int zram_bvec_write(struct zram *zram, struct bio_vec *bvec, u32 index, + int offset) +{ + int ret; + u32 store_offset; + size_t clen; + struct zobj_header *zheader; + struct page *page, *page_store; + unsigned char *user_mem, *cmem, *src, *uncmem = NULL; + + page = bvec->bv_page; + src = zram->compress_buffer; + if (is_partial_io(bvec)) { /* - * System overwrites unused sectors. Free memory associated - * with this sector now. + * This is a partial IO. We need to read the full page + * before to write the changes. */ - if (zram->table[index].page || - zram_test_flag(zram, index, ZRAM_ZERO)) - zram_free_page(zram, index); - - mutex_lock(&zram->lock); - - user_mem = kmap_atomic(page, KM_USER0); - if (page_zero_filled(user_mem)) { - kunmap_atomic(user_mem, KM_USER0); - mutex_unlock(&zram->lock); - zram_stat_inc(&zram->stats.pages_zero); - zram_set_flag(zram, index, ZRAM_ZERO); - index++; - continue; + uncmem = kmalloc(PAGE_SIZE, GFP_KERNEL); + if (!uncmem) { + pr_info("Error allocating temp memory!\n"); + ret = -ENOMEM; + goto out; + } + ret = zram_read_before_write(zram, uncmem, index); + if (ret) { + kfree(uncmem); + goto out; } + } + + /* + * System overwrites unused sectors. Free memory associated + * with this sector now. + */ + if (zram->table[index].page || + zram_test_flag(zram, index, ZRAM_ZERO)) + zram_free_page(zram, index); + + user_mem = kmap_atomic(page, KM_USER0); - ret = lzo1x_1_compress(user_mem, PAGE_SIZE, src, &clen, - zram->compress_workmem); + if (is_partial_io(bvec)) + memcpy(uncmem + offset, user_mem + bvec->bv_offset, + bvec->bv_len); + else + uncmem = user_mem; + if (page_zero_filled(uncmem)) { kunmap_atomic(user_mem, KM_USER0); + if (is_partial_io(bvec)) + kfree(uncmem); + zram_stat_inc(&zram->stats.pages_zero); + zram_set_flag(zram, index, ZRAM_ZERO); + ret = 0; + goto out; + } - if (unlikely(ret != LZO_E_OK)) { - mutex_unlock(&zram->lock); - pr_err("Compression failed! err=%d\n", ret); - zram_stat64_inc(zram, &zram->stats.failed_writes); - goto out; - } + COMPRESS(uncmem, PAGE_SIZE, src, &clen, + zram->compress_workmem); + ret = 0; - /* - * Page is incompressible. Store it as-is (uncompressed) - * since we do not want to return too many disk write - * errors which has side effect of hanging the system. - */ - if (unlikely(clen > max_zpage_size)) { - clen = PAGE_SIZE; - page_store = alloc_page(GFP_NOIO | __GFP_HIGHMEM); - if (unlikely(!page_store)) { - mutex_unlock(&zram->lock); - pr_info("Error allocating memory for " - "incompressible page: %u\n", index); - zram_stat64_inc(zram, - &zram->stats.failed_writes); - goto out; - } - - offset = 0; - zram_set_flag(zram, index, ZRAM_UNCOMPRESSED); - zram_stat_inc(&zram->stats.pages_expand); - zram->table[index].page = page_store; - src = kmap_atomic(page, KM_USER0); - goto memstore; - } + kunmap_atomic(user_mem, KM_USER0); + if (is_partial_io(bvec)) + kfree(uncmem); + + if (unlikely(ret != 0)) { + pr_err("Compression failed! err=%d\n", ret); + goto out; + } - if (xv_malloc(zram->mem_pool, clen + sizeof(*zheader), - &zram->table[index].page, &offset, - GFP_NOIO | __GFP_HIGHMEM)) { - mutex_unlock(&zram->lock); - pr_info("Error allocating memory for compressed " - "page: %u, size=%zu\n", index, clen); - zram_stat64_inc(zram, &zram->stats.failed_writes); + /* + * Page is incompressible. Store it as-is (uncompressed) + * since we do not want to return too many disk write + * errors which has side effect of hanging the system. + */ + if (unlikely(clen > max_zpage_size)) { + clen = PAGE_SIZE; + page_store = alloc_page(GFP_NOIO | __GFP_HIGHMEM); + if (unlikely(!page_store)) { + pr_info("Error allocating memory for " + "incompressible page: %u\n", index); + ret = -ENOMEM; goto out; } + store_offset = 0; + zram_set_flag(zram, index, ZRAM_UNCOMPRESSED); + zram_stat_inc(&zram->stats.pages_expand); + zram->table[index].page = page_store; + src = kmap_atomic(page, KM_USER0); + goto memstore; + } + + if (xv_malloc(zram->mem_pool, clen + sizeof(*zheader), + &zram->table[index].page, &store_offset, + GFP_NOIO | __GFP_HIGHMEM)) { + pr_info("Error allocating memory for compressed " + "page: %u, size=%zu\n", index, clen); + ret = -ENOMEM; + goto out; + } + memstore: - zram->table[index].offset = offset; + zram->table[index].offset = store_offset; - cmem = kmap_atomic(zram->table[index].page, KM_USER1) + - zram->table[index].offset; + cmem = kmap_atomic(zram->table[index].page, KM_USER1) + + zram->table[index].offset; #if 0 - /* Back-reference needed for memory defragmentation */ - if (!zram_test_flag(zram, index, ZRAM_UNCOMPRESSED)) { - zheader = (struct zobj_header *)cmem; - zheader->table_idx = index; - cmem += sizeof(*zheader); - } + /* Back-reference needed for memory defragmentation */ + if (!zram_test_flag(zram, index, ZRAM_UNCOMPRESSED)) { + zheader = (struct zobj_header *)cmem; + zheader->table_idx = index; + cmem += sizeof(*zheader); + } #endif - memcpy(cmem, src, clen); + memcpy(cmem, src, clen); - kunmap_atomic(cmem, KM_USER1); - if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) - kunmap_atomic(src, KM_USER0); + kunmap_atomic(cmem, KM_USER1); + if (unlikely(zram_test_flag(zram, index, ZRAM_UNCOMPRESSED))) + kunmap_atomic(src, KM_USER0); + + /* Update stats */ + zram_stat64_add(zram, &zram->stats.compr_size, clen); + zram_stat_inc(&zram->stats.pages_stored); + if (clen <= PAGE_SIZE / 2) + zram_stat_inc(&zram->stats.good_compress); - /* Update stats */ - zram_stat64_add(zram, &zram->stats.compr_size, clen); - zram_stat_inc(&zram->stats.pages_stored); - if (clen <= PAGE_SIZE / 2) - zram_stat_inc(&zram->stats.good_compress); + return 0; - mutex_unlock(&zram->lock); - index++; +out: + if (ret) + zram_stat64_inc(zram, &zram->stats.failed_writes); + return ret; +} + +static int zram_bvec_rw(struct zram *zram, struct bio_vec *bvec, u32 index, + int offset, struct bio *bio, int rw) +{ + int ret; + + if (rw == READ) { + down_read(&zram->lock); + ret = zram_bvec_read(zram, bvec, index, offset, bio); + up_read(&zram->lock); + } else { + down_write(&zram->lock); + ret = zram_bvec_write(zram, bvec, index, offset); + up_write(&zram->lock); + } + + return ret; +} + +static void update_position(u32 *index, int *offset, struct bio_vec *bvec) +{ + if (*offset + bvec->bv_len >= PAGE_SIZE) + (*index)++; + *offset = (*offset + bvec->bv_len) % PAGE_SIZE; +} + +static void __zram_make_request(struct zram *zram, struct bio *bio, int rw) +{ + int i, offset; + u32 index; + struct bio_vec *bvec; + + switch (rw) { + case READ: + zram_stat64_inc(zram, &zram->stats.num_reads); + break; + case WRITE: + zram_stat64_inc(zram, &zram->stats.num_writes); + break; + } + + index = bio->bi_sector >> SECTORS_PER_PAGE_SHIFT; + offset = (bio->bi_sector & (SECTORS_PER_PAGE - 1)) << SECTOR_SHIFT; + + bio_for_each_segment(bvec, bio, i) { + int max_transfer_size = PAGE_SIZE - offset; + + if (bvec->bv_len > max_transfer_size) { + /* + * zram_bvec_rw() can only make operation on a single + * zram page. Split the bio vector. + */ + struct bio_vec bv; + + bv.bv_page = bvec->bv_page; + bv.bv_len = max_transfer_size; + bv.bv_offset = bvec->bv_offset; + + if (zram_bvec_rw(zram, &bv, index, offset, bio, rw) < 0) + goto out; + + bv.bv_len = bvec->bv_len - max_transfer_size; + bv.bv_offset += max_transfer_size; + if (zram_bvec_rw(zram, &bv, index+1, 0, bio, rw) < 0) + goto out; + } else + if (zram_bvec_rw(zram, bvec, index, offset, bio, rw) + < 0) + goto out; + + update_position(&index, &offset, bvec); } set_bit(BIO_UPTODATE, &bio->bi_flags); bio_endio(bio, 0); - return 0; + return; out: bio_io_error(bio); - return 0; } /* - * Check if request is within bounds and page aligned. + * Check if request is within bounds and aligned on zram logical blocks. */ static inline int valid_io_request(struct zram *zram, struct bio *bio) { if (unlikely( (bio->bi_sector >= (zram->disksize >> SECTOR_SHIFT)) || - (bio->bi_sector & (SECTORS_PER_PAGE - 1)) || - (bio->bi_size & (PAGE_SIZE - 1)))) { + (bio->bi_sector & (ZRAM_SECTOR_PER_LOGICAL_BLOCK - 1)) || + (bio->bi_size & (ZRAM_LOGICAL_BLOCK_SIZE - 1)))) { return 0; } @@ -436,33 +590,36 @@ static inline int valid_io_request(struct zram *zram, struct bio *bio) */ static int zram_make_request(struct request_queue *queue, struct bio *bio) { - int ret = 0; struct zram *zram = queue->queuedata; + if (unlikely(!zram->init_done) && zram_init_device(zram)) + goto error; + + down_read(&zram->init_lock); + if (unlikely(!zram->init_done)) + goto error_unlock; + if (!valid_io_request(zram, bio)) { zram_stat64_inc(zram, &zram->stats.invalid_io); - bio_io_error(bio); - return 0; + goto error_unlock; } - switch (bio_data_dir(bio)) { - case READ: - ret = zram_read(zram, bio); - break; + __zram_make_request(zram, bio, bio_data_dir(bio)); + up_read(&zram->init_lock); - case WRITE: - ret = zram_write(zram, bio); - break; - } + return 0; - return ret; +error_unlock: + up_read(&zram->init_lock); +error: + bio_io_error(bio); + return 0; } -void zram_reset_device(struct zram *zram) +void __zram_reset_device(struct zram *zram) { size_t index; - mutex_lock(&zram->init_lock); zram->init_done = 0; /* Free various per-device buffers */ @@ -498,8 +655,14 @@ void zram_reset_device(struct zram *zram) /* Reset stats */ memset(&zram->stats, 0, sizeof(zram->stats)); - zram->disksize = 0; - mutex_unlock(&zram->init_lock); + zram_set_disksize(zram, zram_default_disksize_bytes()); +} + +void zram_reset_device(struct zram *zram) +{ + down_write(&zram->init_lock); + __zram_reset_device(zram); + up_write(&zram->init_lock); } int zram_init_device(struct zram *zram) @@ -507,41 +670,35 @@ int zram_init_device(struct zram *zram) int ret; size_t num_pages; - mutex_lock(&zram->init_lock); + down_write(&zram->init_lock); if (zram->init_done) { - mutex_unlock(&zram->init_lock); + up_write(&zram->init_lock); return 0; } - zram_set_disksize(zram, totalram_pages << PAGE_SHIFT); - - zram->compress_workmem = kzalloc(LZO1X_MEM_COMPRESS, GFP_KERNEL); + zram->compress_workmem = kzalloc(WMSIZE, GFP_KERNEL); if (!zram->compress_workmem) { pr_err("Error allocating compressor working memory!\n"); ret = -ENOMEM; - goto fail; + goto fail_no_table; } - zram->compress_buffer = (void *)__get_free_pages(__GFP_ZERO, 1); + zram->compress_buffer = + (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, 1); if (!zram->compress_buffer) { pr_err("Error allocating compressor buffer space\n"); ret = -ENOMEM; - goto fail; + goto fail_no_table; } num_pages = zram->disksize >> PAGE_SHIFT; - zram->table = vmalloc(num_pages * sizeof(*zram->table)); + zram->table = vzalloc(num_pages * sizeof(*zram->table)); if (!zram->table) { pr_err("Error allocating zram address table\n"); - /* To prevent accessing table entries during cleanup */ - zram->disksize = 0; ret = -ENOMEM; - goto fail; + goto fail_no_table; } - memset(zram->table, 0, num_pages * sizeof(*zram->table)); - - set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT); /* zram devices sort of resembles non-rotational disks */ queue_flag_set_unlocked(QUEUE_FLAG_NONROT, zram->disk->queue); @@ -554,20 +711,23 @@ int zram_init_device(struct zram *zram) } zram->init_done = 1; - mutex_unlock(&zram->init_lock); + up_write(&zram->init_lock); pr_debug("Initialization done!\n"); return 0; +fail_no_table: + /* To prevent accessing table entries during cleanup */ + zram->disksize = 0; fail: - mutex_unlock(&zram->init_lock); - zram_reset_device(zram); - + __zram_reset_device(zram); + up_write(&zram->init_lock); pr_err("Initialization failed: err=%d\n", ret); return ret; } -void zram_slot_free_notify(struct block_device *bdev, unsigned long index) +static void zram_slot_free_notify(struct block_device *bdev, + unsigned long index) { struct zram *zram; @@ -585,8 +745,8 @@ static int create_device(struct zram *zram, int device_id) { int ret = 0; - mutex_init(&zram->lock); - mutex_init(&zram->init_lock); + init_rwsem(&zram->lock); + init_rwsem(&zram->init_lock); spin_lock_init(&zram->stat64_lock); zram->queue = blk_alloc_queue(GFP_KERNEL); @@ -617,28 +777,31 @@ static int create_device(struct zram *zram, int device_id) zram->disk->private_data = zram; snprintf(zram->disk->disk_name, 16, "zram%d", device_id); - /* Actual capacity set using syfs (/sys/block/zram/disksize */ - set_capacity(zram->disk, 0); + /* + * Set some default disksize. To set another disksize, user + * must reset the device and then write a new disksize to + * corresponding device's sysfs node. + */ + zram_set_disksize(zram, zram_default_disksize_bytes()); /* * To ensure that we always get PAGE_SIZE aligned * and n*PAGE_SIZED sized I/O requests. */ blk_queue_physical_block_size(zram->disk->queue, PAGE_SIZE); - blk_queue_logical_block_size(zram->disk->queue, PAGE_SIZE); + blk_queue_logical_block_size(zram->disk->queue, + ZRAM_LOGICAL_BLOCK_SIZE); blk_queue_io_min(zram->disk->queue, PAGE_SIZE); blk_queue_io_opt(zram->disk->queue, PAGE_SIZE); add_disk(zram->disk); -#ifdef CONFIG_SYSFS ret = sysfs_create_group(&disk_to_dev(zram->disk)->kobj, &zram_disk_attr_group); if (ret < 0) { pr_warning("Error creating sysfs group"); goto out; } -#endif zram->init_done = 0; @@ -648,10 +811,8 @@ static int create_device(struct zram *zram, int device_id) static void destroy_device(struct zram *zram) { -#ifdef CONFIG_SYSFS sysfs_remove_group(&disk_to_dev(zram->disk)->kobj, &zram_disk_attr_group); -#endif if (zram->disk) { del_gendisk(zram->disk); @@ -666,9 +827,16 @@ static int __init zram_init(void) { int ret, dev_id; - if (num_devices > max_num_devices) { + /* + * Module parameter not specified by user. Use default + * value as defined during kernel config. + */ + if (zram_num_devices == 0) + zram_num_devices = CONFIG_ZRAM_NUM_DEVICES; + + if (zram_num_devices > max_num_devices) { pr_warning("Invalid value for num_devices: %u\n", - num_devices); + zram_num_devices); ret = -EINVAL; goto out; } @@ -680,21 +848,18 @@ static int __init zram_init(void) goto out; } - if (!num_devices) { - pr_info("num_devices not specified. Using default: 1\n"); - num_devices = 1; - } - /* Allocate the device array and initialize each one */ - pr_info("Creating %u devices ...\n", num_devices); - devices = kzalloc(num_devices * sizeof(struct zram), GFP_KERNEL); - if (!devices) { + pr_info("Creating %u devices ...\n", zram_num_devices); + zram_devices = kzalloc(zram_num_devices * sizeof(struct zram), + GFP_KERNEL); + if (!zram_devices) + { ret = -ENOMEM; goto unregister; } - for (dev_id = 0; dev_id < num_devices; dev_id++) { - ret = create_device(&devices[dev_id], dev_id); + for (dev_id = 0; dev_id < zram_num_devices; dev_id++) { + ret = create_device(&zram_devices[dev_id], dev_id); if (ret) goto free_devices; } @@ -703,8 +868,8 @@ static int __init zram_init(void) free_devices: while (dev_id) - destroy_device(&devices[--dev_id]); - kfree(devices); + destroy_device(&zram_devices[--dev_id]); + kfree(zram_devices); unregister: unregister_blkdev(zram_major, "zram"); out: @@ -716,8 +881,8 @@ static void __exit zram_exit(void) int i; struct zram *zram; - for (i = 0; i < num_devices; i++) { - zram = &devices[i]; + for (i = 0; i < zram_num_devices; i++) { + zram = &zram_devices[i]; destroy_device(zram); if (zram->init_done) @@ -726,11 +891,11 @@ static void __exit zram_exit(void) unregister_blkdev(zram_major, "zram"); - kfree(devices); + kfree(zram_devices); pr_debug("Cleanup done!\n"); } -module_param(num_devices, uint, 0); +module_param_named(num_devices, zram_num_devices, uint, 0); MODULE_PARM_DESC(num_devices, "Number of zram devices"); module_init(zram_init); diff --git a/kernel/drivers/staging/zram/zram_drv.h b/kernel/drivers/staging/zram/zram_drv.h index a4815511..36986381 100644 --- a/kernel/drivers/staging/zram/zram_drv.h +++ b/kernel/drivers/staging/zram/zram_drv.h @@ -41,13 +41,13 @@ struct zobj_header { /*-- Configurable parameters */ /* Default zram disk size: 25% of total RAM */ -static const unsigned default_disksize_perc_ram = 25; +static const unsigned default_disksize_perc_ram = CONFIG_ZRAM_DEFAULT_PERCENTAGE; /* * Pages that compress to size greater than this are stored * uncompressed in memory. */ -static const unsigned max_zpage_size = PAGE_SIZE / 4 * 3; +static const size_t max_zpage_size = PAGE_SIZE / 4 * 3; /* * NOTE: max_zpage_size must be less than or equal to: @@ -61,6 +61,10 @@ static const unsigned max_zpage_size = PAGE_SIZE / 4 * 3; #define SECTOR_SIZE (1 << SECTOR_SHIFT) #define SECTORS_PER_PAGE_SHIFT (PAGE_SHIFT - SECTOR_SHIFT) #define SECTORS_PER_PAGE (1 << SECTORS_PER_PAGE_SHIFT) +#define ZRAM_LOGICAL_BLOCK_SHIFT 12 +#define ZRAM_LOGICAL_BLOCK_SIZE (1 << ZRAM_LOGICAL_BLOCK_SHIFT) +#define ZRAM_SECTOR_PER_LOGICAL_BLOCK \ + (1 << (ZRAM_LOGICAL_BLOCK_SHIFT - SECTOR_SHIFT)) /* Flags for zram pages (table[page_no].flags) */ enum zram_pageflags { @@ -103,13 +107,13 @@ struct zram { void *compress_buffer; struct table *table; spinlock_t stat64_lock; /* protect 64-bit stats */ - struct mutex lock; /* protect compression buffers against - * concurrent writes */ + struct rw_semaphore lock; /* protect compression buffers and table + * against concurrent read and writes */ struct request_queue *queue; struct gendisk *disk; int init_done; - /* Prevent concurrent execution of device init and reset */ - struct mutex init_lock; + /* Prevent concurrent execution of device init, reset and R/W request */ + struct rw_semaphore init_lock; /* * This is the limit on amount of *uncompressed* worth of data * we can store in a disk. @@ -119,13 +123,13 @@ struct zram { struct zram_stats stats; }; -extern struct zram *devices; -extern unsigned int num_devices; +extern struct zram *zram_devices; +extern unsigned int zram_num_devices; #ifdef CONFIG_SYSFS extern struct attribute_group zram_disk_attr_group; #endif extern int zram_init_device(struct zram *zram); -extern void zram_reset_device(struct zram *zram); +extern void __zram_reset_device(struct zram *zram); #endif diff --git a/kernel/drivers/staging/zram/zram_sysfs.c b/kernel/drivers/staging/zram/zram_sysfs.c index 6b3cf00b..0ea8ed29 100644 --- a/kernel/drivers/staging/zram/zram_sysfs.c +++ b/kernel/drivers/staging/zram/zram_sysfs.c @@ -14,11 +14,10 @@ #include #include +#include #include "zram_drv.h" -#ifdef CONFIG_SYSFS - static u64 zram_stat64_read(struct zram *zram, u64 *v) { u64 val; @@ -35,8 +34,8 @@ static struct zram *dev_to_zram(struct device *dev) int i; struct zram *zram = NULL; - for (i = 0; i < num_devices; i++) { - zram = &devices[i]; + for (i = 0; i < zram_num_devices; i++) { + zram = &zram_devices[i]; if (disk_to_dev(zram->disk) == dev) break; } @@ -56,19 +55,23 @@ static ssize_t disksize_store(struct device *dev, struct device_attribute *attr, const char *buf, size_t len) { int ret; + u64 disksize; struct zram *zram = dev_to_zram(dev); + ret = strict_strtoull(buf, 10, &disksize); + if (ret) + return ret; + + down_write(&zram->init_lock); if (zram->init_done) { + up_write(&zram->init_lock); pr_info("Cannot change disksize for initialized device\n"); return -EBUSY; } - ret = strict_strtoull(buf, 10, &zram->disksize); - if (ret) - return ret; - - zram->disksize &= PAGE_MASK; + zram->disksize = PAGE_ALIGN(disksize); set_capacity(zram->disk, zram->disksize >> SECTOR_SHIFT); + up_write(&zram->init_lock); return len; } @@ -107,8 +110,10 @@ static ssize_t reset_store(struct device *dev, if (bdev) fsync_bdev(bdev); + down_write(&zram->init_lock); if (zram->init_done) - zram_reset_device(zram); + __zram_reset_device(zram); + up_write(&zram->init_lock); return len; } @@ -220,5 +225,3 @@ static struct attribute *zram_disk_attrs[] = { struct attribute_group zram_disk_attr_group = { .attrs = zram_disk_attrs, }; - -#endif /* CONFIG_SYSFS */ diff --git a/kernel/include/linux/vmalloc.h b/kernel/include/linux/vmalloc.h index 227c2a58..6c4a7988 100755 --- a/kernel/include/linux/vmalloc.h +++ b/kernel/include/linux/vmalloc.h @@ -51,6 +51,7 @@ static inline void vmalloc_init(void) #endif extern void *vmalloc(unsigned long size); +extern void *vzalloc(unsigned long size); extern void *vmalloc_user(unsigned long size); extern void *vmalloc_node(unsigned long size, int node); extern void *vmalloc_exec(unsigned long size); diff --git a/kernel/mm/vmalloc.c b/kernel/mm/vmalloc.c index ae007462..c3bca468 100755 --- a/kernel/mm/vmalloc.c +++ b/kernel/mm/vmalloc.c @@ -1583,6 +1583,13 @@ void *__vmalloc(unsigned long size, gfp_t gfp_mask, pgprot_t prot) } EXPORT_SYMBOL(__vmalloc); +static inline void *__vmalloc_node_flags(unsigned long size, + int node, gfp_t flags) +{ + return __vmalloc_node(size, 1, flags, PAGE_KERNEL, + node, __builtin_return_address(0)); +} + /** * vmalloc - allocate virtually contiguous memory * @size: allocation size @@ -1599,6 +1606,24 @@ void *vmalloc(unsigned long size) } EXPORT_SYMBOL(vmalloc); +/** + * vzalloc - allocate virtually contiguous memory with zero fill + * @size: allocation size + * Allocate enough pages to cover @size from the page level + * allocator and map them into contiguous kernel virtual space. + * The memory allocated is set to zero. + * + * For tight control over page level allocator and protection flags + * use __vmalloc() instead. + */ +void *vzalloc(unsigned long size) +{ + return __vmalloc_node_flags(size, -1, + GFP_KERNEL | __GFP_HIGHMEM | __GFP_ZERO); +} +EXPORT_SYMBOL(vzalloc); + + /** * vmalloc_user - allocate zeroed virtually contiguous memory for userspace * @size: allocation size