From 75f80aabd728aafec6adc8d586be800b95229cb1 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Thu, 10 Nov 2022 17:43:26 -0600 Subject: [PATCH 1/9] Added block_size search in lfs_mount, moved block_size/count into lfs_t This adds two new configuration options: erase_size and erase_count, allowing block_size and block_count to be loaded from the superblock during mount. For backwards compability these default to block_size and block_count if zero. --- Unfortunately this is a bit easier said than done. littlefs keeps its superblock in the metadata pair located at blocks {0,1}, which is also where the root directory lives (keep in mind small littlefs images may only have 2 blocks in total). If we mutate blocks {0,1}, we have to erase before programming, which means it's possible to have the only superblock in block 1. This presents a puzzle because how do you find block 1 if you don't know the size of a block? One solution presented here is to search for block 1 by trying different sizes until we find a superblock. This isn't great but there are some properties of this search that help: 1. If we do find a superblock, the search will never take longer than a mount with a known block_size. This is because we stop at block 1, searching at most O(block_size) bytes, and metadata fetch is already a O(block_size) operation. This means the concern is limited to how long it takes to fail when littlefs is not present on the disk. 2. We can assume the on-disk block_size is probably a factor of the total size of the disk. After a bit of digging into the math, this apparently reduces the runtime to the divisor function, d(n), which is sublinear. According to a blog post by Terence Tao this is bounded by the ridiculous O(e^O(log(n)/log(log(n)))): https://terrytao.wordpress.com/2008/09/23/the-divisor-bound This is apparently somewhere between O(sqrt(n)) and O(log(n)), but conveniently O(log(n)) on average and O(log(n)) for powers of 2. I've left it as O(d(n)) in the documentation, which might be a bit confusing, but I'm not sure how best to capture "mostly log(n)" correctly. 3. If we don't know the block_size, or don't know that block_size is aligned to the disk size, the best we can do is a O(n) search. In this case I've added a warning, so at least it's distinguishable from an infinite loop if debugging. --- lfs.c | 472 +++++++++++++++++++++++++----------- lfs.h | 53 +++- tests/test_badblocks.toml | 2 +- tests/test_evil.toml | 2 +- tests/test_superblocks.toml | 2 +- 5 files changed, 374 insertions(+), 157 deletions(-) diff --git a/lfs.c b/lfs.c index 48a10e8e..63be7fb8 100644 --- a/lfs.c +++ b/lfs.c @@ -26,6 +26,100 @@ enum { }; +/// Mapping from logical to physical erase size /// + +static int lfs_bd_rawread(lfs_t *lfs, lfs_block_t block, + lfs_off_t off, void *buffer, lfs_size_t size) { + LFS_ASSERT(block < lfs->block_count); + LFS_ASSERT(off + size <= lfs->block_size); + LFS_ASSERT(size % lfs->cfg->read_size == 0); + + // adjust to physical erase size + block = (block * (lfs->block_size/lfs->erase_size)) + + (off / lfs->erase_size); + off = off % lfs->erase_size; + uint8_t *buffer_ = buffer; + + // read in erase_size chunks + while (size > 0) { + lfs_size_t delta = lfs_min(size, off + lfs->erase_size); + LFS_ASSERT(block <= lfs->erase_count); + LFS_ASSERT(off + size <= lfs->erase_size); + LFS_ASSERT(size % lfs->cfg->read_size == 0); + int err = lfs->cfg->read(lfs->cfg, block, off, buffer_, delta); + LFS_ASSERT(err <= 0); + if (err) { + return err; + } + + off += delta; + if (off == lfs->erase_size) { + block += 1; + off = 0; + } + size -= delta; + } + + return 0; +} + +#ifndef LFS_READONLY +static int lfs_bd_rawprog(lfs_t *lfs, lfs_block_t block, + lfs_off_t off, void *buffer, lfs_size_t size) { + LFS_ASSERT(block < lfs->block_count); + LFS_ASSERT(off + size <= lfs->block_size); + LFS_ASSERT(size % lfs->cfg->prog_size == 0); + + // adjust to physical erase size + block = (block * (lfs->block_size/lfs->erase_size)) + + (off / lfs->erase_size); + off = off % lfs->erase_size; + uint8_t *buffer_ = buffer; + + // prog in erase_size chunks + while (size > 0) { + lfs_size_t delta = lfs_min(size, off + lfs->erase_size); + LFS_ASSERT(block <= lfs->erase_count); + LFS_ASSERT(off + size <= lfs->erase_size); + LFS_ASSERT(size % lfs->cfg->prog_size == 0); + int err = lfs->cfg->prog(lfs->cfg, block, off, buffer_, delta); + LFS_ASSERT(err <= 0); + if (err) { + return err; + } + + off += delta; + if (off == lfs->erase_size) { + block += 1; + off = 0; + } + size -= delta; + } + + return 0; +} +#endif + +#ifndef LFS_READONLY +static int lfs_bd_erase(lfs_t *lfs, lfs_block_t block) { + LFS_ASSERT(block < lfs->block_count); + + // adjust to physical erase size + block = block * (lfs->block_size/lfs->erase_size); + for (lfs_block_t i = 0; i < lfs->block_size/lfs->erase_size; i++) { + LFS_ASSERT(block + i <= lfs->erase_count); + int err = lfs->cfg->erase(lfs->cfg, block + i); + LFS_ASSERT(err <= 0); + if (err < 0) { + return err; + } + } + + return 0; +} +#endif + + /// Caching block device operations /// static inline void lfs_cache_drop(lfs_t *lfs, lfs_cache_t *rcache) { @@ -46,8 +140,8 @@ static int lfs_bd_read(lfs_t *lfs, lfs_block_t block, lfs_off_t off, void *buffer, lfs_size_t size) { uint8_t *data = buffer; - if (block >= lfs->cfg->block_count || - off+size > lfs->cfg->block_size) { + if (block >= lfs->block_count || + off+size > lfs->block_size) { return LFS_ERR_CORRUPT; } @@ -92,7 +186,7 @@ static int lfs_bd_read(lfs_t *lfs, size >= lfs->cfg->read_size) { // bypass cache? diff = lfs_aligndown(diff, lfs->cfg->read_size); - int err = lfs->cfg->read(lfs->cfg, block, off, data, diff); + int err = lfs_bd_rawread(lfs, block, off, data, diff); if (err) { return err; } @@ -104,18 +198,16 @@ static int lfs_bd_read(lfs_t *lfs, } // load to cache, first condition can no longer fail - LFS_ASSERT(block < lfs->cfg->block_count); rcache->block = block; rcache->off = lfs_aligndown(off, lfs->cfg->read_size); rcache->size = lfs_min( lfs_min( lfs_alignup(off+hint, lfs->cfg->read_size), - lfs->cfg->block_size) + lfs->block_size) - rcache->off, lfs->cfg->cache_size); - int err = lfs->cfg->read(lfs->cfg, rcache->block, + int err = lfs_bd_rawread(lfs, rcache->block, rcache->off, rcache->buffer, rcache->size); - LFS_ASSERT(err <= 0); if (err) { return err; } @@ -155,11 +247,9 @@ static int lfs_bd_cmp(lfs_t *lfs, static int lfs_bd_flush(lfs_t *lfs, lfs_cache_t *pcache, lfs_cache_t *rcache, bool validate) { if (pcache->block != LFS_BLOCK_NULL && pcache->block != LFS_BLOCK_INLINE) { - LFS_ASSERT(pcache->block < lfs->cfg->block_count); lfs_size_t diff = lfs_alignup(pcache->size, lfs->cfg->prog_size); - int err = lfs->cfg->prog(lfs->cfg, pcache->block, + int err = lfs_bd_rawprog(lfs, pcache->block, pcache->off, pcache->buffer, diff); - LFS_ASSERT(err <= 0); if (err) { return err; } @@ -208,8 +298,8 @@ static int lfs_bd_prog(lfs_t *lfs, lfs_block_t block, lfs_off_t off, const void *buffer, lfs_size_t size) { const uint8_t *data = buffer; - LFS_ASSERT(block == LFS_BLOCK_INLINE || block < lfs->cfg->block_count); - LFS_ASSERT(off + size <= lfs->cfg->block_size); + LFS_ASSERT(block == LFS_BLOCK_INLINE || block < lfs->block_count); + LFS_ASSERT(off + size <= lfs->block_size); while (size > 0) { if (block == pcache->block && @@ -250,15 +340,6 @@ static int lfs_bd_prog(lfs_t *lfs, } #endif -#ifndef LFS_READONLY -static int lfs_bd_erase(lfs_t *lfs, lfs_block_t block) { - LFS_ASSERT(block < lfs->cfg->block_count); - int err = lfs->cfg->erase(lfs->cfg, block); - LFS_ASSERT(err <= 0); - return err; -} -#endif - /// Small type-level utilities /// // operations on block pairs @@ -528,7 +609,7 @@ static int lfs_rawunmount(lfs_t *lfs); static int lfs_alloc_lookahead(void *p, lfs_block_t block) { lfs_t *lfs = (lfs_t*)p; lfs_block_t off = ((block - lfs->free.off) - + lfs->cfg->block_count) % lfs->cfg->block_count; + + lfs->block_count) % lfs->block_count; if (off < lfs->free.size) { lfs->free.buffer[off / 32] |= 1U << (off % 32); @@ -542,7 +623,7 @@ static int lfs_alloc_lookahead(void *p, lfs_block_t block) { // is to prevent blocks from being garbage collected in the middle of a // commit operation static void lfs_alloc_ack(lfs_t *lfs) { - lfs->free.ack = lfs->cfg->block_count; + lfs->free.ack = lfs->block_count; } // drop the lookahead buffer, this is done during mounting and failed @@ -563,7 +644,7 @@ static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { if (!(lfs->free.buffer[off / 32] & (1U << (off % 32)))) { // found a free block - *block = (lfs->free.off + off) % lfs->cfg->block_count; + *block = (lfs->free.off + off) % lfs->block_count; // eagerly find next off so an alloc ack can // discredit old lookahead blocks @@ -586,7 +667,7 @@ static int lfs_alloc(lfs_t *lfs, lfs_block_t *block) { } lfs->free.off = (lfs->free.off + lfs->free.size) - % lfs->cfg->block_count; + % lfs->block_count; lfs->free.size = lfs_min(8*lfs->cfg->lookahead_size, lfs->free.ack); lfs->free.i = 0; @@ -676,7 +757,7 @@ static int lfs_dir_getread(lfs_t *lfs, const lfs_mdir_t *dir, lfs_tag_t gmask, lfs_tag_t gtag, lfs_off_t off, void *buffer, lfs_size_t size) { uint8_t *data = buffer; - if (off+size > lfs->cfg->block_size) { + if (off+size > lfs->block_size) { return LFS_ERR_CORRUPT; } @@ -998,7 +1079,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, // if either block address is invalid we return LFS_ERR_CORRUPT here, // otherwise later writes to the pair could fail - if (pair[0] >= lfs->cfg->block_count || pair[1] >= lfs->cfg->block_count) { + if (pair[0] >= lfs->block_count || pair[1] >= lfs->block_count) { return LFS_ERR_CORRUPT; } @@ -1044,7 +1125,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, lfs_tag_t tag; off += lfs_tag_dsize(ptag); int err = lfs_bd_read(lfs, - NULL, &lfs->rcache, lfs->cfg->block_size, + NULL, &lfs->rcache, lfs->block_size, dir->pair[0], off, &tag, sizeof(tag)); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -1063,7 +1144,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, dir->erased = (lfs_tag_type1(ptag) == LFS_TYPE_CRC && dir->off % lfs->cfg->prog_size == 0); break; - } else if (off + lfs_tag_dsize(tag) > lfs->cfg->block_size) { + } else if (off + lfs_tag_dsize(tag) > lfs->block_size) { dir->erased = false; break; } @@ -1074,7 +1155,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, // check the crc attr uint32_t dcrc; err = lfs_bd_read(lfs, - NULL, &lfs->rcache, lfs->cfg->block_size, + NULL, &lfs->rcache, lfs->block_size, dir->pair[0], off+sizeof(tag), &dcrc, sizeof(dcrc)); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -1117,7 +1198,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, for (lfs_off_t j = sizeof(tag); j < lfs_tag_dsize(tag); j++) { uint8_t dat; err = lfs_bd_read(lfs, - NULL, &lfs->rcache, lfs->cfg->block_size, + NULL, &lfs->rcache, lfs->block_size, dir->pair[0], off+j, &dat, 1); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -1150,7 +1231,7 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, tempsplit = (lfs_tag_chunk(tag) & 1); err = lfs_bd_read(lfs, - NULL, &lfs->rcache, lfs->cfg->block_size, + NULL, &lfs->rcache, lfs->block_size, dir->pair[0], off+sizeof(tag), &temptail, 8); if (err) { if (err == LFS_ERR_CORRUPT) { @@ -1770,7 +1851,7 @@ static int lfs_dir_compact(lfs_t *lfs, .begin = 0, .end = (lfs->cfg->metadata_max ? - lfs->cfg->metadata_max : lfs->cfg->block_size) - 8, + lfs->cfg->metadata_max : lfs->block_size) - 8, }; // erase block to write to @@ -1940,11 +2021,11 @@ static int lfs_dir_splittingcompact(lfs_t *lfs, lfs_mdir_t *dir, // if (end - split < 0xff && size <= lfs_min( - lfs->cfg->block_size - 40, + lfs->block_size - 40, lfs_alignup( (lfs->cfg->metadata_max ? lfs->cfg->metadata_max - : lfs->cfg->block_size)/2, + : lfs->block_size)/2, lfs->cfg->prog_size))) { break; } @@ -1986,7 +2067,7 @@ static int lfs_dir_splittingcompact(lfs_t *lfs, lfs_mdir_t *dir, // do we have extra space? littlefs can't reclaim this space // by itself, so expand cautiously - if ((lfs_size_t)size < lfs->cfg->block_count/2) { + if ((lfs_size_t)size < lfs->block_count/2) { LFS_DEBUG("Expanding superblock at rev %"PRIu32, dir->rev); int err = lfs_dir_split(lfs, dir, attrs, attrcount, source, begin, end); @@ -2056,7 +2137,7 @@ static int lfs_dir_relocatingcommit(lfs_t *lfs, lfs_mdir_t *dir, .begin = dir->off, .end = (lfs->cfg->metadata_max ? - lfs->cfg->metadata_max : lfs->cfg->block_size) - 8, + lfs->cfg->metadata_max : lfs->block_size) - 8, }; // traverse attrs that need to be written out @@ -2647,7 +2728,7 @@ static int lfs_dir_rawrewind(lfs_t *lfs, lfs_dir_t *dir) { /// File index list operations /// static int lfs_ctz_index(lfs_t *lfs, lfs_off_t *off) { lfs_off_t size = *off; - lfs_off_t b = lfs->cfg->block_size - 2*4; + lfs_off_t b = lfs->block_size - 2*4; lfs_off_t i = size / b; if (i == 0) { return 0; @@ -2725,7 +2806,7 @@ static int lfs_ctz_extend(lfs_t *lfs, noff = noff + 1; // just copy out the last block if it is incomplete - if (noff != lfs->cfg->block_size) { + if (noff != lfs->block_size) { for (lfs_off_t i = 0; i < noff; i++) { uint8_t data; err = lfs_bd_read(lfs, @@ -3265,7 +3346,7 @@ static lfs_ssize_t lfs_file_flushedread(lfs_t *lfs, lfs_file_t *file, while (nsize > 0) { // check if we need a new block if (!(file->flags & LFS_F_READING) || - file->off == lfs->cfg->block_size) { + file->off == lfs->block_size) { if (!(file->flags & LFS_F_INLINE)) { int err = lfs_ctz_find(lfs, NULL, &file->cache, file->ctz.head, file->ctz.size, @@ -3282,10 +3363,10 @@ static lfs_ssize_t lfs_file_flushedread(lfs_t *lfs, lfs_file_t *file, } // read as much as we can in current block - lfs_size_t diff = lfs_min(nsize, lfs->cfg->block_size - file->off); + lfs_size_t diff = lfs_min(nsize, lfs->block_size - file->off); if (file->flags & LFS_F_INLINE) { int err = lfs_dir_getread(lfs, &file->m, - NULL, &file->cache, lfs->cfg->block_size, + NULL, &file->cache, lfs->block_size, LFS_MKTAG(0xfff, 0x1ff, 0), LFS_MKTAG(LFS_TYPE_INLINESTRUCT, file->id, 0), file->off, data, diff); @@ -3294,7 +3375,7 @@ static lfs_ssize_t lfs_file_flushedread(lfs_t *lfs, lfs_file_t *file, } } else { int err = lfs_bd_read(lfs, - NULL, &file->cache, lfs->cfg->block_size, + NULL, &file->cache, lfs->block_size, file->block, file->off, data, diff); if (err) { return err; @@ -3339,7 +3420,7 @@ static lfs_ssize_t lfs_file_flushedwrite(lfs_t *lfs, lfs_file_t *file, lfs_min(0x3fe, lfs_min( lfs->cfg->cache_size, (lfs->cfg->metadata_max ? - lfs->cfg->metadata_max : lfs->cfg->block_size) / 8))) { + lfs->cfg->metadata_max : lfs->block_size) / 8))) { // inline file doesn't fit anymore int err = lfs_file_outline(lfs, file); if (err) { @@ -3351,7 +3432,7 @@ static lfs_ssize_t lfs_file_flushedwrite(lfs_t *lfs, lfs_file_t *file, while (nsize > 0) { // check if we need a new block if (!(file->flags & LFS_F_WRITING) || - file->off == lfs->cfg->block_size) { + file->off == lfs->block_size) { if (!(file->flags & LFS_F_INLINE)) { if (!(file->flags & LFS_F_WRITING) && file->pos > 0) { // find out which block we're extending from @@ -3385,7 +3466,7 @@ static lfs_ssize_t lfs_file_flushedwrite(lfs_t *lfs, lfs_file_t *file, } // program as much as we can in current block - lfs_size_t diff = lfs_min(nsize, lfs->cfg->block_size - file->off); + lfs_size_t diff = lfs_min(nsize, lfs->block_size - file->off); while (true) { int err = lfs_bd_prog(lfs, &file->cache, &lfs->rcache, true, file->block, file->off, data, diff); @@ -3915,20 +3996,40 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { int err = 0; // validate that the lfs-cfg sizes were initiated properly before - // performing any arithmetic logics with them + // performing any arithmetic logic with them LFS_ASSERT(lfs->cfg->read_size != 0); LFS_ASSERT(lfs->cfg->prog_size != 0); LFS_ASSERT(lfs->cfg->cache_size != 0); + LFS_ASSERT(lfs->cfg->erase_size != 0 || lfs->cfg->block_size != 0); + LFS_ASSERT(lfs->cfg->erase_count != 0 || lfs->cfg->block_count != 0); - // check that block size is a multiple of cache size is a multiple - // of prog and read sizes + // check that cache_size is a multiple of prog_size and read_size LFS_ASSERT(lfs->cfg->cache_size % lfs->cfg->read_size == 0); LFS_ASSERT(lfs->cfg->cache_size % lfs->cfg->prog_size == 0); - LFS_ASSERT(lfs->cfg->block_size % lfs->cfg->cache_size == 0); + + // setup erase_size/count, these can be zero for backwards compatibility + lfs->erase_size = lfs->cfg->erase_size; + lfs->erase_count = lfs->cfg->erase_count; + lfs->block_size = lfs->cfg->block_size; + lfs->block_count = lfs->cfg->block_count; + if (!lfs->erase_size) { + lfs->erase_size = lfs->block_size; + } + if (!lfs->erase_count) { + lfs->erase_count = lfs->block_count; + } + + // check that block_size is a multiple of erase_size is a mulitiple of + // cache_size, this implies everything is a multiple of read_size and + // prog_size + LFS_ASSERT(lfs->erase_size % lfs->cfg->cache_size == 0); + if (lfs->block_size) { + LFS_ASSERT(lfs->block_size % lfs->erase_size == 0); + } // check that the block size is large enough to fit ctz pointers - LFS_ASSERT(4*lfs_npw2(0xffffffff / (lfs->cfg->block_size-2*4)) - <= lfs->cfg->block_size); + LFS_ASSERT(4*lfs_npw2(0xffffffff / (lfs->block_size-2*4)) + <= lfs->block_size); // block_cycles = 0 is no longer supported. // @@ -3998,7 +4099,7 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->attr_max = LFS_ATTR_MAX; } - LFS_ASSERT(lfs->cfg->metadata_max <= lfs->cfg->block_size); + LFS_ASSERT(lfs->cfg->metadata_max <= lfs->block_size); // setup default state lfs->root[0] = LFS_BLOCK_NULL; @@ -4045,11 +4146,19 @@ static int lfs_rawformat(lfs_t *lfs, const struct lfs_config *cfg) { return err; } + // if block_size/block_count not specified, assume equal to erase blocks + if (!lfs->block_size) { + lfs->block_size = lfs->erase_size; + } + if (!lfs->block_count) { + lfs->block_count = lfs->erase_count; + } + // create free lookahead memset(lfs->free.buffer, 0, lfs->cfg->lookahead_size); lfs->free.off = 0; lfs->free.size = lfs_min(8*lfs->cfg->lookahead_size, - lfs->cfg->block_count); + lfs->block_count); lfs->free.i = 0; lfs_alloc_ack(lfs); @@ -4063,8 +4172,8 @@ static int lfs_rawformat(lfs_t *lfs, const struct lfs_config *cfg) { // write one superblock lfs_superblock_t superblock = { .version = LFS_DISK_VERSION, - .block_size = lfs->cfg->block_size, - .block_count = lfs->cfg->block_count, + .block_size = lfs->block_size, + .block_count = lfs->block_count, .name_max = lfs->name_max, .file_max = lfs->file_max, .attr_max = lfs->attr_max, @@ -4108,111 +4217,180 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - // scan directory blocks for superblock and any global updates - lfs_mdir_t dir = {.tail = {0, 1}}; - lfs_block_t cycle = 0; - while (!lfs_pair_isnull(dir.tail)) { - if (cycle >= lfs->cfg->block_count/2) { - // loop detected - err = LFS_ERR_CORRUPT; - goto cleanup; - } - cycle += 1; - - // fetch next block in tail list - lfs_stag_t tag = lfs_dir_fetchmatch(lfs, &dir, dir.tail, - LFS_MKTAG(0x7ff, 0x3ff, 0), - LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), - NULL, - lfs_dir_find_match, &(struct lfs_dir_find_match){ - lfs, "littlefs", 8}); - if (tag < 0) { - err = tag; - goto cleanup; + // if block_size is unknown we need to search for it + lfs_size_t block_size_limit = lfs->block_size; + if (!lfs->block_size) { + lfs->block_size = lfs->erase_size; + // make sure this doesn't overflow + lfs_size_t limit = lfs->block_count + ? lfs->block_count/2 + : lfs->erase_count/2; + if (limit > ((lfs_size_t)-1) / lfs->block_size) { + block_size_limit = ((lfs_size_t)-1); + } else { + block_size_limit = limit * lfs->block_size; } + } - // has superblock? - if (tag && !lfs_tag_isdelete(tag)) { - // update root - lfs->root[0] = dir.pair[0]; - lfs->root[1] = dir.pair[1]; + // search for the correct block_size + while (true) { + // setup block_size/count so underlying operations work + lfs->block_count = lfs->erase_count + / (lfs->block_size/lfs->erase_size); + + // scan directory blocks for superblock and any global updates + lfs_mdir_t dir = {.tail = {0, 1}}; + lfs_block_t cycle = 0; + while (!lfs_pair_isnull(dir.tail)) { + if (cycle >= lfs->block_count/2) { + // loop detected + err = LFS_ERR_CORRUPT; + goto cleanup; + } + cycle += 1; - // grab superblock - lfs_superblock_t superblock; - tag = lfs_dir_get(lfs, &dir, LFS_MKTAG(0x7ff, 0x3ff, 0), - LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), - &superblock); + // fetch next block in tail list + lfs_stag_t tag = lfs_dir_fetchmatch(lfs, &dir, dir.tail, + LFS_MKTAG(0x7ff, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), + NULL, + lfs_dir_find_match, &(struct lfs_dir_find_match){ + lfs, "littlefs", 8}); if (tag < 0) { + if (tag == LFS_ERR_CORRUPT) { + // maybe our block_size is wrong + goto next_block_size; + } err = tag; goto cleanup; } - lfs_superblock_fromle32(&superblock); - - // check version - uint16_t major_version = (0xffff & (superblock.version >> 16)); - uint16_t minor_version = (0xffff & (superblock.version >> 0)); - if ((major_version != LFS_DISK_VERSION_MAJOR || - minor_version > LFS_DISK_VERSION_MINOR)) { - LFS_ERROR("Invalid version v%"PRIu16".%"PRIu16, - major_version, minor_version); - err = LFS_ERR_INVAL; - goto cleanup; - } - // check superblock configuration - if (superblock.name_max) { - if (superblock.name_max > lfs->name_max) { - LFS_ERROR("Unsupported name_max (%"PRIu32" > %"PRIu32")", - superblock.name_max, lfs->name_max); - err = LFS_ERR_INVAL; + // has superblock? + if (tag && !lfs_tag_isdelete(tag)) { + // grab superblock + lfs_superblock_t superblock; + tag = lfs_dir_get(lfs, &dir, LFS_MKTAG(0x7ff, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), + &superblock); + if (tag < 0) { + if (tag == LFS_ERR_CORRUPT) { + // maybe our block_size is wrong + goto next_block_size; + } + err = tag; goto cleanup; } + lfs_superblock_fromle32(&superblock); + + // we may not be done, first check the stored block_size, it + // it's different we need to remount in case we found an + // outdated superblock + if (superblock.block_size != lfs->block_size) { + if (lfs->cfg->block_size + || superblock.block_size % lfs->erase_size != 0 + || superblock.block_size < lfs->block_size) { + LFS_ERROR("Invalid block size %"PRIu32, + superblock.block_size); + err = LFS_ERR_INVAL; + goto cleanup; + } + + // remount with correct block_size + lfs->block_size = superblock.block_size; + goto next_mount; + } - lfs->name_max = superblock.name_max; - } + if (superblock.block_count != lfs->block_count) { + if ((lfs->cfg->block_size && lfs->cfg->block_count) + || superblock.block_count > lfs->block_count) { + LFS_ERROR("Invalid block count %"PRIu32, + superblock.block_count); + err = LFS_ERR_INVAL; + goto cleanup; + } + + lfs->block_count = superblock.block_count; + } - if (superblock.file_max) { - if (superblock.file_max > lfs->file_max) { - LFS_ERROR("Unsupported file_max (%"PRIu32" > %"PRIu32")", - superblock.file_max, lfs->file_max); + // check version + uint16_t major_version = (0xffff & (superblock.version >> 16)); + uint16_t minor_version = (0xffff & (superblock.version >> 0)); + if ((major_version != LFS_DISK_VERSION_MAJOR || + minor_version > LFS_DISK_VERSION_MINOR)) { + LFS_ERROR("Invalid version v%"PRIu16".%"PRIu16, + major_version, minor_version); err = LFS_ERR_INVAL; goto cleanup; } - lfs->file_max = superblock.file_max; - } + // check superblock configuration + if (superblock.name_max) { + if (superblock.name_max > lfs->name_max) { + LFS_ERROR("Unsupported name_max %"PRIu32, + superblock.name_max); + err = LFS_ERR_INVAL; + goto cleanup; + } - if (superblock.attr_max) { - if (superblock.attr_max > lfs->attr_max) { - LFS_ERROR("Unsupported attr_max (%"PRIu32" > %"PRIu32")", - superblock.attr_max, lfs->attr_max); - err = LFS_ERR_INVAL; - goto cleanup; + lfs->name_max = superblock.name_max; } - lfs->attr_max = superblock.attr_max; - } + if (superblock.file_max) { + if (superblock.file_max > lfs->file_max) { + LFS_ERROR("Unsupported file_max %"PRIu32, + superblock.file_max); + err = LFS_ERR_INVAL; + goto cleanup; + } - if (superblock.block_count != lfs->cfg->block_count) { - LFS_ERROR("Invalid block count (%"PRIu32" != %"PRIu32")", - superblock.block_count, lfs->cfg->block_count); - err = LFS_ERR_INVAL; - goto cleanup; + lfs->file_max = superblock.file_max; + } + + if (superblock.attr_max) { + if (superblock.attr_max > lfs->attr_max) { + LFS_ERROR("Unsupported attr_max %"PRIu32, + superblock.attr_max); + err = LFS_ERR_INVAL; + goto cleanup; + } + + lfs->attr_max = superblock.attr_max; + } + + // update root + lfs->root[0] = dir.pair[0]; + lfs->root[1] = dir.pair[1]; } - if (superblock.block_size != lfs->cfg->block_size) { - LFS_ERROR("Invalid block size (%"PRIu32" != %"PRIu32")", - superblock.block_size, lfs->cfg->block_size); - err = LFS_ERR_INVAL; + // has gstate? + err = lfs_dir_getgstate(lfs, &dir, &lfs->gstate); + if (err) { goto cleanup; } + + // we found a valid superblock, set block_size_limit so block_size + // will no longer change + block_size_limit = lfs->block_size; } - // has gstate? - err = lfs_dir_getgstate(lfs, &dir, &lfs->gstate); - if (err) { + break; + +next_block_size: + lfs->block_size += lfs->erase_size; + if (lfs->block_size > block_size_limit) { + err = LFS_ERR_INVAL; goto cleanup; } + + // if block_count is set, skip block_sizes that aren't a factor, + // this brings our search down from O(n) to O(d(n)), and + // O(log(n)) for powers of 2 + if (lfs->cfg->block_count && lfs->cfg->block_count + % (lfs->block_size/lfs->erase_size) != 0) { + goto next_block_size; + } + +next_mount:; } // found superblock? @@ -4233,7 +4411,7 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) { // setup free lookahead, to distribute allocations uniformly across // boots, we start the allocator at a random location - lfs->free.off = lfs->seed % lfs->cfg->block_count; + lfs->free.off = lfs->seed % lfs->block_count; lfs_alloc_drop(lfs); return 0; @@ -4270,7 +4448,7 @@ int lfs_fs_rawtraverse(lfs_t *lfs, lfs_block_t cycle = 0; while (!lfs_pair_isnull(dir.tail)) { - if (cycle >= lfs->cfg->block_count/2) { + if (cycle >= lfs->block_count/2) { // loop detected return LFS_ERR_CORRUPT; } @@ -4355,7 +4533,7 @@ static int lfs_fs_pred(lfs_t *lfs, pdir->tail[1] = 1; lfs_block_t cycle = 0; while (!lfs_pair_isnull(pdir->tail)) { - if (cycle >= lfs->cfg->block_count/2) { + if (cycle >= lfs->block_count/2) { // loop detected return LFS_ERR_CORRUPT; } @@ -4392,7 +4570,7 @@ static int lfs_fs_parent_match(void *data, lfs_block_t child[2]; int err = lfs_bd_read(lfs, - &lfs->pcache, &lfs->rcache, lfs->cfg->block_size, + &lfs->pcache, &lfs->rcache, lfs->block_size, disk->block, disk->off, &child, sizeof(child)); if (err) { return err; @@ -4411,7 +4589,7 @@ static lfs_stag_t lfs_fs_parent(lfs_t *lfs, const lfs_block_t pair[2], parent->tail[1] = 1; lfs_block_t cycle = 0; while (!lfs_pair_isnull(parent->tail)) { - if (cycle >= lfs->cfg->block_count/2) { + if (cycle >= lfs->block_count/2) { // loop detected return LFS_ERR_CORRUPT; } @@ -4828,7 +5006,7 @@ static int lfs1_dir_fetch(lfs_t *lfs, } if ((0x7fffffff & test.size) < sizeof(test)+4 || - (0x7fffffff & test.size) > lfs->cfg->block_size) { + (0x7fffffff & test.size) > lfs->block_size) { continue; } @@ -5262,8 +5440,8 @@ static int lfs_rawmigrate(lfs_t *lfs, const struct lfs_config *cfg) { lfs_superblock_t superblock = { .version = LFS_DISK_VERSION, - .block_size = lfs->cfg->block_size, - .block_count = lfs->cfg->block_count, + .block_size = lfs->block_size, + .block_count = lfs->block_count, .name_max = lfs->name_max, .file_max = lfs->file_max, .attr_max = lfs->attr_max, @@ -5324,6 +5502,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { LFS_TRACE("lfs_format(%p, %p {.context=%p, " ".read=%p, .prog=%p, .erase=%p, .sync=%p, " ".read_size=%"PRIu32", .prog_size=%"PRIu32", " + ".erase_size=%"PRIu32", .erase_count=%"PRIu32", " ".block_size=%"PRIu32", .block_count=%"PRIu32", " ".block_cycles=%"PRIu32", .cache_size=%"PRIu32", " ".lookahead_size=%"PRIu32", .read_buffer=%p, " @@ -5333,8 +5512,9 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { (void*)lfs, (void*)cfg, cfg->context, (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, - cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, - cfg->block_cycles, cfg->cache_size, cfg->lookahead_size, + cfg->read_size, cfg->prog_size, cfg->erase_size, cfg->erase_count, + cfg->block_size, cfg->block_count, cfg->block_cycles, + cfg->cache_size, cfg->lookahead_size, cfg->read_buffer, cfg->prog_buffer, cfg->lookahead_buffer, cfg->name_max, cfg->file_max, cfg->attr_max); @@ -5354,6 +5534,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { LFS_TRACE("lfs_mount(%p, %p {.context=%p, " ".read=%p, .prog=%p, .erase=%p, .sync=%p, " ".read_size=%"PRIu32", .prog_size=%"PRIu32", " + ".erase_size=%"PRIu32", .erase_count=%"PRIu32", " ".block_size=%"PRIu32", .block_count=%"PRIu32", " ".block_cycles=%"PRIu32", .cache_size=%"PRIu32", " ".lookahead_size=%"PRIu32", .read_buffer=%p, " @@ -5363,8 +5544,9 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { (void*)lfs, (void*)cfg, cfg->context, (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, - cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, - cfg->block_cycles, cfg->cache_size, cfg->lookahead_size, + cfg->read_size, cfg->prog_size, cfg->erase_size, cfg->erase_count, + cfg->block_size, cfg->block_count, cfg->block_cycles, + cfg->cache_size, cfg->lookahead_size, cfg->read_buffer, cfg->prog_buffer, cfg->lookahead_buffer, cfg->name_max, cfg->file_max, cfg->attr_max); diff --git a/lfs.h b/lfs.h index ab534526..d5927bca 100644 --- a/lfs.h +++ b/lfs.h @@ -189,21 +189,52 @@ struct lfs_config { int (*unlock)(const struct lfs_config *c); #endif - // Minimum size of a block read in bytes. All read operations will be a - // multiple of this value. + // Minimum size of a read operation in bytes. All read operations + // will be a multiple of this value. lfs_size_t read_size; - // Minimum size of a block program in bytes. All program operations will be - // a multiple of this value. + // Minimum size of a program operation in bytes. All program operations + // will be a multiple of this value. lfs_size_t prog_size; - // Size of an erasable block in bytes. This does not impact ram consumption - // and may be larger than the physical erase size. However, non-inlined - // files take up at minimum one block. Must be a multiple of the read and - // program sizes. + // Minimum size of an erase operation in bytes. All erase operations + // will be a multiple of this value. This must be a multiple of the read + // and program sizes. + // + // If zero, the block_size is used as the erase_size. This is mostly for + // backwards compatibility. + lfs_size_t erase_size; + + // Number of erase blocks on the device. + // + // If zero, the block_count is used as the erase_count. This is mostly for + // backwards compatibility. + lfs_size_t erase_count; + + // Size of a logical block in bytes. This does not impact RAM consumption + // and may be a multiple of the physical erase_size. + // + // Note, non-inlined files take up at minimum one logical block, and + // directories take up at minimum two logical blocks. + // + // If zero, littlefs attempts to find the superblock and use the the + // block_size stored there. This requires searching different block_sizes. + // If a superblock is found this takes no longer than mounting with a known + // block_size, but it can take time to fail if a superblock is not found: + // + // - O(block_size) if a superblock is found + // - O(d(block_count)) if block_count is non-zero + // - O(log(block_count)) if block_count is a power of 2 + // - O(erase_count) if block_count is zero lfs_size_t block_size; - // Number of erasable blocks on the device. + // Number of logical blocks on the device. + // + // If zero, littlefs uses the block_count stored in the superblock. + // + // If non-zero and block_size is zero, littlefs will assume block_size + // is a factor of erase_size*block_count to speed up mount when no + // superblock is found. lfs_size_t block_count; // Number of erase cycles before littlefs evicts metadata logs and moves @@ -408,6 +439,10 @@ typedef struct lfs { } free; const struct lfs_config *cfg; + lfs_size_t erase_size; + lfs_size_t erase_count; + lfs_size_t block_size; + lfs_size_t block_count; lfs_size_t name_max; lfs_size_t file_max; lfs_size_t attr_max; diff --git a/tests/test_badblocks.toml b/tests/test_badblocks.toml index b50b3933..5035b931 100644 --- a/tests/test_badblocks.toml +++ b/tests/test_badblocks.toml @@ -256,5 +256,5 @@ code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => LFS_ERR_NOSPC; - lfs_mount(&lfs, cfg) => LFS_ERR_CORRUPT; + lfs_mount(&lfs, cfg) => LFS_ERR_INVAL; ''' diff --git a/tests/test_evil.toml b/tests/test_evil.toml index 4acd5ef0..1dde8650 100644 --- a/tests/test_evil.toml +++ b/tests/test_evil.toml @@ -24,7 +24,7 @@ code = ''' lfs_deinit(&lfs) => 0; // test that mount fails gracefully - lfs_mount(&lfs, cfg) => LFS_ERR_CORRUPT; + lfs_mount(&lfs, cfg) => LFS_ERR_INVAL; ''' [cases.test_evil_invalid_dir_pointer] diff --git a/tests/test_superblocks.toml b/tests/test_superblocks.toml index 689bbcd2..60bda767 100644 --- a/tests/test_superblocks.toml +++ b/tests/test_superblocks.toml @@ -31,7 +31,7 @@ code = ''' [cases.test_superblocks_invalid_mount] code = ''' lfs_t lfs; - lfs_mount(&lfs, cfg) => LFS_ERR_CORRUPT; + lfs_mount(&lfs, cfg) => LFS_ERR_INVAL; ''' # expanding superblock From 03beff6c54b1e3e47f03bc3512379e5cb22b0752 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 11 Nov 2022 00:53:44 -0600 Subject: [PATCH 2/9] Changed block_size search to use erase_count when block_size is zero This hopefully helps make it more clear that we are using erase_size as the unit during the block_size search. However this does mean you need to set erase_count to zero to allow any littlefs size, which might be a bit confusing: block_size block_count erase_size erase_count known known known known => known bs, O(1) known unknown known known => known bs, O(1) unknown unknown known known => bs search, O(d(n)) unknown unknown known unknown => bs search, O(n) --- lfs.c | 25 +++++++++++++------------ lfs.h | 15 ++++++++------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/lfs.c b/lfs.c index 63be7fb8..55276b22 100644 --- a/lfs.c +++ b/lfs.c @@ -4001,7 +4001,6 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { LFS_ASSERT(lfs->cfg->prog_size != 0); LFS_ASSERT(lfs->cfg->cache_size != 0); LFS_ASSERT(lfs->cfg->erase_size != 0 || lfs->cfg->block_size != 0); - LFS_ASSERT(lfs->cfg->erase_count != 0 || lfs->cfg->block_count != 0); // check that cache_size is a multiple of prog_size and read_size LFS_ASSERT(lfs->cfg->cache_size % lfs->cfg->read_size == 0); @@ -4222,21 +4221,23 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) { if (!lfs->block_size) { lfs->block_size = lfs->erase_size; // make sure this doesn't overflow - lfs_size_t limit = lfs->block_count - ? lfs->block_count/2 - : lfs->erase_count/2; - if (limit > ((lfs_size_t)-1) / lfs->block_size) { + if (!lfs->erase_count || lfs->erase_count/2 + > ((lfs_size_t)-1) / lfs->erase_size) { block_size_limit = ((lfs_size_t)-1); } else { - block_size_limit = limit * lfs->block_size; + block_size_limit = (lfs->erase_count/2) * lfs->erase_size; } } // search for the correct block_size while (true) { // setup block_size/count so underlying operations work - lfs->block_count = lfs->erase_count - / (lfs->block_size/lfs->erase_size); + if (!lfs->erase_count) { + lfs->block_count = (lfs_size_t)-1; + } else { + lfs->block_count = lfs->erase_count + / (lfs->block_size/lfs->erase_size); + } // scan directory blocks for superblock and any global updates lfs_mdir_t dir = {.tail = {0, 1}}; @@ -4301,7 +4302,7 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) { } if (superblock.block_count != lfs->block_count) { - if ((lfs->cfg->block_size && lfs->cfg->block_count) + if (lfs->cfg->block_count || superblock.block_count > lfs->block_count) { LFS_ERROR("Invalid block count %"PRIu32, superblock.block_count); @@ -4383,9 +4384,9 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) { } // if block_count is set, skip block_sizes that aren't a factor, - // this brings our search down from O(n) to O(d(n)), and - // O(log(n)) for powers of 2 - if (lfs->cfg->block_count && lfs->cfg->block_count + // this brings our search down from O(n) to O(d(n)), O(log(n)) + // on average, and O(log(n)) for powers of 2 + if (lfs->erase_count && lfs->erase_count % (lfs->block_size/lfs->erase_size) != 0) { goto next_block_size; } diff --git a/lfs.h b/lfs.h index d5927bca..6b2780ed 100644 --- a/lfs.h +++ b/lfs.h @@ -209,6 +209,11 @@ struct lfs_config { // // If zero, the block_count is used as the erase_count. This is mostly for // backwards compatibility. + // + // If zero and block_count is zero, this is treated as unknown. + // + // If non-zero, littlefs will assume block_size is a factor of + // erase_size*erase_count to speed up mount when no superblock is found. lfs_size_t erase_count; // Size of a logical block in bytes. This does not impact RAM consumption @@ -223,18 +228,14 @@ struct lfs_config { // block_size, but it can take time to fail if a superblock is not found: // // - O(block_size) if a superblock is found - // - O(d(block_count)) if block_count is non-zero - // - O(log(block_count)) if block_count is a power of 2 - // - O(erase_count) if block_count is zero + // - O(d(erase_count)) if erase_count is non-zero + // - O(log(erase_count)) if erase_count is a power of 2 + // - O(erase_count) if erase_count is zero lfs_size_t block_size; // Number of logical blocks on the device. // // If zero, littlefs uses the block_count stored in the superblock. - // - // If non-zero and block_size is zero, littlefs will assume block_size - // is a factor of erase_size*block_count to speed up mount when no - // superblock is found. lfs_size_t block_count; // Number of erase cycles before littlefs evicts metadata logs and moves From c16120bf5f67259eb91288320400462b688d69eb Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 11 Nov 2022 12:49:36 -0600 Subject: [PATCH 3/9] Dropped erase_count, block_count makes it redundant Aside from asserts (which can be implemented in block devices), erase_count provides no useful info aside from what's known with block_count and risks mistakes during configuration. Each set of known/unknown block_size-related variables has a well-defined mount behavior: block_size block_count known known => known bs, O(1) known unknown => known bs, O(1) unknown known => bs search, O(d(n)) unknown unknown => bs search, O(n) If block_size is unknown, block_count uses the erase_size as a unit, which is the only reasonable option since block_size must be >= prog/read/cache_size. --- lfs.c | 60 ++++++++++++++++++++++++++--------------------------------- lfs.h | 21 ++++++--------------- 2 files changed, 32 insertions(+), 49 deletions(-) diff --git a/lfs.c b/lfs.c index 55276b22..cf44f95f 100644 --- a/lfs.c +++ b/lfs.c @@ -43,7 +43,6 @@ static int lfs_bd_rawread(lfs_t *lfs, lfs_block_t block, // read in erase_size chunks while (size > 0) { lfs_size_t delta = lfs_min(size, off + lfs->erase_size); - LFS_ASSERT(block <= lfs->erase_count); LFS_ASSERT(off + size <= lfs->erase_size); LFS_ASSERT(size % lfs->cfg->read_size == 0); int err = lfs->cfg->read(lfs->cfg, block, off, buffer_, delta); @@ -79,7 +78,6 @@ static int lfs_bd_rawprog(lfs_t *lfs, lfs_block_t block, // prog in erase_size chunks while (size > 0) { lfs_size_t delta = lfs_min(size, off + lfs->erase_size); - LFS_ASSERT(block <= lfs->erase_count); LFS_ASSERT(off + size <= lfs->erase_size); LFS_ASSERT(size % lfs->cfg->prog_size == 0); int err = lfs->cfg->prog(lfs->cfg, block, off, buffer_, delta); @@ -107,7 +105,6 @@ static int lfs_bd_erase(lfs_t *lfs, lfs_block_t block) { // adjust to physical erase size block = block * (lfs->block_size/lfs->erase_size); for (lfs_block_t i = 0; i < lfs->block_size/lfs->erase_size; i++) { - LFS_ASSERT(block + i <= lfs->erase_count); int err = lfs->cfg->erase(lfs->cfg, block + i); LFS_ASSERT(err <= 0); if (err < 0) { @@ -4006,30 +4003,20 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { LFS_ASSERT(lfs->cfg->cache_size % lfs->cfg->read_size == 0); LFS_ASSERT(lfs->cfg->cache_size % lfs->cfg->prog_size == 0); - // setup erase_size/count, these can be zero for backwards compatibility + // setup erase_size, this can be zero for backwards compatibility lfs->erase_size = lfs->cfg->erase_size; - lfs->erase_count = lfs->cfg->erase_count; - lfs->block_size = lfs->cfg->block_size; - lfs->block_count = lfs->cfg->block_count; if (!lfs->erase_size) { - lfs->erase_size = lfs->block_size; - } - if (!lfs->erase_count) { - lfs->erase_count = lfs->block_count; + lfs->erase_size = lfs->cfg->block_size; } // check that block_size is a multiple of erase_size is a mulitiple of // cache_size, this implies everything is a multiple of read_size and // prog_size LFS_ASSERT(lfs->erase_size % lfs->cfg->cache_size == 0); - if (lfs->block_size) { - LFS_ASSERT(lfs->block_size % lfs->erase_size == 0); + if (lfs->cfg->block_size) { + LFS_ASSERT(lfs->cfg->block_size % lfs->erase_size == 0); } - // check that the block size is large enough to fit ctz pointers - LFS_ASSERT(4*lfs_npw2(0xffffffff / (lfs->block_size-2*4)) - <= lfs->block_size); - // block_cycles = 0 is no longer supported. // // block_cycles is the number of erase cycles before littlefs evicts @@ -4145,13 +4132,17 @@ static int lfs_rawformat(lfs_t *lfs, const struct lfs_config *cfg) { return err; } - // if block_size/block_count not specified, assume equal to erase blocks + // if block_size not specified, assume equal to erase blocks + lfs->block_size = lfs->cfg->block_size; if (!lfs->block_size) { lfs->block_size = lfs->erase_size; } - if (!lfs->block_count) { - lfs->block_count = lfs->erase_count; - } + lfs->block_count = lfs->cfg->block_count; + LFS_ASSERT(lfs->block_count != 0); + + // check that the block size is large enough to fit ctz pointers + LFS_ASSERT(4*lfs_npw2(0xffffffff / (lfs->block_size-2*4)) + <= lfs->block_size); // create free lookahead memset(lfs->free.buffer, 0, lfs->cfg->lookahead_size); @@ -4217,26 +4208,27 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) { } // if block_size is unknown we need to search for it - lfs_size_t block_size_limit = lfs->block_size; + lfs->block_size = lfs->cfg->block_size; + lfs_size_t block_size_limit = lfs->cfg->block_size; if (!lfs->block_size) { lfs->block_size = lfs->erase_size; // make sure this doesn't overflow - if (!lfs->erase_count || lfs->erase_count/2 + if (!lfs->cfg->block_count || lfs->cfg->block_count/2 > ((lfs_size_t)-1) / lfs->erase_size) { block_size_limit = ((lfs_size_t)-1); } else { - block_size_limit = (lfs->erase_count/2) * lfs->erase_size; + block_size_limit = (lfs->cfg->block_count/2) * lfs->erase_size; } } // search for the correct block_size while (true) { // setup block_size/count so underlying operations work - if (!lfs->erase_count) { + lfs->block_count = lfs->cfg->block_count; + if (!lfs->block_count) { lfs->block_count = (lfs_size_t)-1; - } else { - lfs->block_count = lfs->erase_count - / (lfs->block_size/lfs->erase_size); + } else if (!lfs->cfg->block_size) { + lfs->block_count /= lfs->block_size/lfs->erase_size; } // scan directory blocks for superblock and any global updates @@ -4383,10 +4375,10 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) { goto cleanup; } - // if block_count is set, skip block_sizes that aren't a factor, + // if block_count is non-zero, skip block_sizes that aren't a factor, // this brings our search down from O(n) to O(d(n)), O(log(n)) // on average, and O(log(n)) for powers of 2 - if (lfs->erase_count && lfs->erase_count + if (lfs->cfg->block_count && lfs->cfg->block_count % (lfs->block_size/lfs->erase_size) != 0) { goto next_block_size; } @@ -5503,7 +5495,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { LFS_TRACE("lfs_format(%p, %p {.context=%p, " ".read=%p, .prog=%p, .erase=%p, .sync=%p, " ".read_size=%"PRIu32", .prog_size=%"PRIu32", " - ".erase_size=%"PRIu32", .erase_count=%"PRIu32", " + ".erase_size=%"PRIu32", " ".block_size=%"PRIu32", .block_count=%"PRIu32", " ".block_cycles=%"PRIu32", .cache_size=%"PRIu32", " ".lookahead_size=%"PRIu32", .read_buffer=%p, " @@ -5513,7 +5505,7 @@ int lfs_format(lfs_t *lfs, const struct lfs_config *cfg) { (void*)lfs, (void*)cfg, cfg->context, (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, - cfg->read_size, cfg->prog_size, cfg->erase_size, cfg->erase_count, + cfg->read_size, cfg->prog_size, cfg->erase_size, cfg->block_size, cfg->block_count, cfg->block_cycles, cfg->cache_size, cfg->lookahead_size, cfg->read_buffer, cfg->prog_buffer, cfg->lookahead_buffer, @@ -5535,7 +5527,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { LFS_TRACE("lfs_mount(%p, %p {.context=%p, " ".read=%p, .prog=%p, .erase=%p, .sync=%p, " ".read_size=%"PRIu32", .prog_size=%"PRIu32", " - ".erase_size=%"PRIu32", .erase_count=%"PRIu32", " + ".erase_size=%"PRIu32", " ".block_size=%"PRIu32", .block_count=%"PRIu32", " ".block_cycles=%"PRIu32", .cache_size=%"PRIu32", " ".lookahead_size=%"PRIu32", .read_buffer=%p, " @@ -5545,7 +5537,7 @@ int lfs_mount(lfs_t *lfs, const struct lfs_config *cfg) { (void*)lfs, (void*)cfg, cfg->context, (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, - cfg->read_size, cfg->prog_size, cfg->erase_size, cfg->erase_count, + cfg->read_size, cfg->prog_size, cfg->erase_size, cfg->block_size, cfg->block_count, cfg->block_cycles, cfg->cache_size, cfg->lookahead_size, cfg->read_buffer, cfg->prog_buffer, cfg->lookahead_buffer, diff --git a/lfs.h b/lfs.h index 6b2780ed..ae609e43 100644 --- a/lfs.h +++ b/lfs.h @@ -205,17 +205,6 @@ struct lfs_config { // backwards compatibility. lfs_size_t erase_size; - // Number of erase blocks on the device. - // - // If zero, the block_count is used as the erase_count. This is mostly for - // backwards compatibility. - // - // If zero and block_count is zero, this is treated as unknown. - // - // If non-zero, littlefs will assume block_size is a factor of - // erase_size*erase_count to speed up mount when no superblock is found. - lfs_size_t erase_count; - // Size of a logical block in bytes. This does not impact RAM consumption // and may be a multiple of the physical erase_size. // @@ -228,14 +217,17 @@ struct lfs_config { // block_size, but it can take time to fail if a superblock is not found: // // - O(block_size) if a superblock is found - // - O(d(erase_count)) if erase_count is non-zero - // - O(log(erase_count)) if erase_count is a power of 2 - // - O(erase_count) if erase_count is zero + // - O(d(block_count)) if block_count is non-zero + // - O(log(block_count)) if block_count is a power of 2 + // - O(block_count) if block_count is zero lfs_size_t block_size; // Number of logical blocks on the device. // // If zero, littlefs uses the block_count stored in the superblock. + // + // If non-zero, littlefs will assume block_size is a factor of + // erase_size*erase_count to speed up mount when no superblock is found. lfs_size_t block_count; // Number of erase cycles before littlefs evicts metadata logs and moves @@ -441,7 +433,6 @@ typedef struct lfs { const struct lfs_config *cfg; lfs_size_t erase_size; - lfs_size_t erase_count; lfs_size_t block_size; lfs_size_t block_count; lfs_size_t name_max; From 21e54ee45899f8b11e91117d5e305b5110a04f81 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 11 Nov 2022 15:05:34 -0600 Subject: [PATCH 4/9] Adopted erase_size config changes in block devices and test runners This ended up needing a bit of API rework since littlefs no longer needs to know the actual erase_count. We can no longer rely on lfs_config to contain all the information necessary to configure the block devices. Changing the lfs_bd_config structs to be required is probably a good idea anyways as it moves us more towards separating the bds from littlefs, though we can't quite get rid of the lfs_config parameter because of the block-device API in lfs_config. Eventually it would be nice to get rid of it, but that would require API breakage. --- bd/lfs_emubd.c | 105 +++++++++++++++--------------------- bd/lfs_emubd.h | 18 +++++-- bd/lfs_filebd.c | 43 ++++++++------- bd/lfs_filebd.h | 21 +++++++- bd/lfs_rambd.c | 65 +++++++++------------- bd/lfs_rambd.h | 19 +++++-- lfs.h | 5 +- runners/bench_runner.c | 29 +++++----- runners/bench_runner.h | 32 ++++++----- runners/test_runner.c | 57 ++++++++++++++------ runners/test_runner.h | 32 ++++++----- tests/test_alloc.toml | 16 +++--- tests/test_badblocks.toml | 6 +-- tests/test_exhaustion.toml | 10 ++-- tests/test_superblocks.toml | 48 +++++++++++++++++ 15 files changed, 303 insertions(+), 203 deletions(-) diff --git a/bd/lfs_emubd.c b/bd/lfs_emubd.c index 97bcf035..5ab8b082 100644 --- a/bd/lfs_emubd.c +++ b/bd/lfs_emubd.c @@ -48,6 +48,7 @@ static void lfs_emubd_decblock(lfs_emubd_block_t *block) { static lfs_emubd_block_t *lfs_emubd_mutblock( const struct lfs_config *cfg, lfs_emubd_block_t **block) { + lfs_emubd_t *bd = cfg->context; lfs_emubd_block_t *block_ = *block; if (block_ && block_->rc == 1) { // rc == 1? can modify @@ -56,13 +57,13 @@ static lfs_emubd_block_t *lfs_emubd_mutblock( } else if (block_) { // rc > 1? need to create a copy lfs_emubd_block_t *nblock = malloc( - sizeof(lfs_emubd_block_t) + cfg->block_size); + sizeof(lfs_emubd_block_t) + bd->cfg->erase_size); if (!nblock) { return NULL; } memcpy(nblock, block_, - sizeof(lfs_emubd_block_t) + cfg->block_size); + sizeof(lfs_emubd_block_t) + bd->cfg->erase_size); nblock->rc = 1; lfs_emubd_decblock(block_); @@ -72,7 +73,7 @@ static lfs_emubd_block_t *lfs_emubd_mutblock( } else { // no block? need to allocate lfs_emubd_block_t *nblock = malloc( - sizeof(lfs_emubd_block_t) + cfg->block_size); + sizeof(lfs_emubd_block_t) + bd->cfg->erase_size); if (!nblock) { return NULL; } @@ -81,10 +82,9 @@ static lfs_emubd_block_t *lfs_emubd_mutblock( nblock->wear = 0; // zero for consistency - lfs_emubd_t *bd = cfg->context; memset(nblock->data, (bd->cfg->erase_value != -1) ? bd->cfg->erase_value : 0, - cfg->block_size); + bd->cfg->erase_size); *block = nblock; return nblock; @@ -94,22 +94,22 @@ static lfs_emubd_block_t *lfs_emubd_mutblock( // emubd create/destroy -int lfs_emubd_createcfg(const struct lfs_config *cfg, const char *path, +int lfs_emubd_create(const struct lfs_config *cfg, const struct lfs_emubd_config *bdcfg) { - LFS_EMUBD_TRACE("lfs_emubd_createcfg(%p {.context=%p, " - ".read=%p, .prog=%p, .erase=%p, .sync=%p, " - ".read_size=%"PRIu32", .prog_size=%"PRIu32", " - ".block_size=%"PRIu32", .block_count=%"PRIu32"}, " - "\"%s\", " - "%p {.erase_value=%"PRId32", .erase_cycles=%"PRIu32", " + LFS_EMUBD_TRACE("lfs_emubd_create(%p {.context=%p, " + ".read=%p, .prog=%p, .erase=%p, .sync=%p}, " + "%p {.read_size=%"PRIu32", .prog_size=%"PRIu32", " + ".erase_size=%"PRIu32", .erase_count=%"PRIu32", " + ".erase_value=%"PRId32", .erase_cycles=%"PRIu32", " ".badblock_behavior=%"PRIu8", .power_cycles=%"PRIu32", " ".powerloss_behavior=%"PRIu8", .powerloss_cb=%p, " ".powerloss_data=%p, .track_branches=%d})", (void*)cfg, cfg->context, (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, - cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, - path, (void*)bdcfg, bdcfg->erase_value, bdcfg->erase_cycles, + (void*)bdcfg, + bdcfg->read_size, bdcfg->prog_size, bdcfg->erase_size, + bdcfg->erase_count, bdcfg->erase_value, bdcfg->erase_cycles, bdcfg->badblock_behavior, bdcfg->power_cycles, bdcfg->powerloss_behavior, (void*)(uintptr_t)bdcfg->powerloss_cb, bdcfg->powerloss_data, bdcfg->track_branches); @@ -117,12 +117,12 @@ int lfs_emubd_createcfg(const struct lfs_config *cfg, const char *path, bd->cfg = bdcfg; // allocate our block array, all blocks start as uninitialized - bd->blocks = malloc(cfg->block_count * sizeof(lfs_emubd_block_t*)); + bd->blocks = malloc(bd->cfg->erase_count * sizeof(lfs_emubd_block_t*)); if (!bd->blocks) { - LFS_EMUBD_TRACE("lfs_emubd_createcfg -> %d", LFS_ERR_NOMEM); + LFS_EMUBD_TRACE("lfs_emubd_create -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } - memset(bd->blocks, 0, cfg->block_count * sizeof(lfs_emubd_block_t*)); + memset(bd->blocks, 0, bd->cfg->erase_count * sizeof(lfs_emubd_block_t*)); // setup testing things bd->readed = 0; @@ -134,7 +134,7 @@ int lfs_emubd_createcfg(const struct lfs_config *cfg, const char *path, if (bd->cfg->disk_path) { bd->disk = malloc(sizeof(lfs_emubd_disk_t)); if (!bd->disk) { - LFS_EMUBD_TRACE("lfs_emubd_createcfg -> %d", LFS_ERR_NOMEM); + LFS_EMUBD_TRACE("lfs_emubd_create -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } bd->disk->rc = 1; @@ -156,21 +156,21 @@ int lfs_emubd_createcfg(const struct lfs_config *cfg, const char *path, // if we're emulating erase values, we can keep a block around in // memory of just the erase state to speed up emulated erases if (bd->cfg->erase_value != -1) { - bd->disk->scratch = malloc(cfg->block_size); + bd->disk->scratch = malloc(bd->cfg->erase_size); if (!bd->disk->scratch) { - LFS_EMUBD_TRACE("lfs_emubd_createcfg -> %d", LFS_ERR_NOMEM); + LFS_EMUBD_TRACE("lfs_emubd_create -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } memset(bd->disk->scratch, bd->cfg->erase_value, - cfg->block_size); + bd->cfg->erase_size); // go ahead and erase all of the disk, otherwise the file will not // match our internal representation - for (size_t i = 0; i < cfg->block_count; i++) { + for (size_t i = 0; i < bd->cfg->erase_count; i++) { ssize_t res = write(bd->disk->fd, bd->disk->scratch, - cfg->block_size); + bd->cfg->erase_size); if (res < 0) { int err = -errno; LFS_EMUBD_TRACE("lfs_emubd_create -> %d", err); @@ -180,33 +180,16 @@ int lfs_emubd_createcfg(const struct lfs_config *cfg, const char *path, } } - LFS_EMUBD_TRACE("lfs_emubd_createcfg -> %d", 0); + LFS_EMUBD_TRACE("lfs_emubd_create -> %d", 0); return 0; } -int lfs_emubd_create(const struct lfs_config *cfg, const char *path) { - LFS_EMUBD_TRACE("lfs_emubd_create(%p {.context=%p, " - ".read=%p, .prog=%p, .erase=%p, .sync=%p, " - ".read_size=%"PRIu32", .prog_size=%"PRIu32", " - ".block_size=%"PRIu32", .block_count=%"PRIu32"}, " - "\"%s\")", - (void*)cfg, cfg->context, - (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, - (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, - cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, - path); - static const struct lfs_emubd_config defaults = {.erase_value=-1}; - int err = lfs_emubd_createcfg(cfg, path, &defaults); - LFS_EMUBD_TRACE("lfs_emubd_create -> %d", err); - return err; -} - int lfs_emubd_destroy(const struct lfs_config *cfg) { LFS_EMUBD_TRACE("lfs_emubd_destroy(%p)", (void*)cfg); lfs_emubd_t *bd = cfg->context; // decrement reference counts - for (lfs_block_t i = 0; i < cfg->block_count; i++) { + for (lfs_block_t i = 0; i < bd->cfg->erase_count; i++) { lfs_emubd_decblock(bd->blocks[i]); } free(bd->blocks); @@ -237,10 +220,10 @@ int lfs_emubd_read(const struct lfs_config *cfg, lfs_block_t block, lfs_emubd_t *bd = cfg->context; // check if read is valid - LFS_ASSERT(block < cfg->block_count); - LFS_ASSERT(off % cfg->read_size == 0); - LFS_ASSERT(size % cfg->read_size == 0); - LFS_ASSERT(off+size <= cfg->block_size); + LFS_ASSERT(block < bd->cfg->erase_count); + LFS_ASSERT(off % bd->cfg->read_size == 0); + LFS_ASSERT(size % bd->cfg->read_size == 0); + LFS_ASSERT(off+size <= bd->cfg->erase_size); // get the block const lfs_emubd_block_t *b = bd->blocks[block]; @@ -287,10 +270,10 @@ int lfs_emubd_prog(const struct lfs_config *cfg, lfs_block_t block, lfs_emubd_t *bd = cfg->context; // check if write is valid - LFS_ASSERT(block < cfg->block_count); - LFS_ASSERT(off % cfg->prog_size == 0); - LFS_ASSERT(size % cfg->prog_size == 0); - LFS_ASSERT(off+size <= cfg->block_size); + LFS_ASSERT(block < bd->cfg->erase_count); + LFS_ASSERT(off % bd->cfg->prog_size == 0); + LFS_ASSERT(size % bd->cfg->prog_size == 0); + LFS_ASSERT(off+size <= bd->cfg->erase_size); // get the block lfs_emubd_block_t *b = lfs_emubd_mutblock(cfg, &bd->blocks[block]); @@ -327,7 +310,7 @@ int lfs_emubd_prog(const struct lfs_config *cfg, lfs_block_t block, // mirror to disk file? if (bd->disk) { off_t res1 = lseek(bd->disk->fd, - (off_t)block*cfg->block_size + (off_t)off, + (off_t)block*bd->cfg->erase_size + (off_t)off, SEEK_SET); if (res1 < 0) { int err = -errno; @@ -372,11 +355,11 @@ int lfs_emubd_prog(const struct lfs_config *cfg, lfs_block_t block, int lfs_emubd_erase(const struct lfs_config *cfg, lfs_block_t block) { LFS_EMUBD_TRACE("lfs_emubd_erase(%p, 0x%"PRIx32" (%"PRIu32"))", - (void*)cfg, block, cfg->block_size); + (void*)cfg, block, ((lfs_emubd_t*)cfg->context)->cfg->erase_size); lfs_emubd_t *bd = cfg->context; // check if erase is valid - LFS_ASSERT(block < cfg->block_count); + LFS_ASSERT(block < bd->cfg->erase_count); // get the block lfs_emubd_block_t *b = lfs_emubd_mutblock(cfg, &bd->blocks[block]); @@ -405,12 +388,12 @@ int lfs_emubd_erase(const struct lfs_config *cfg, lfs_block_t block) { // emulate an erase value? if (bd->cfg->erase_value != -1) { - memset(b->data, bd->cfg->erase_value, cfg->block_size); + memset(b->data, bd->cfg->erase_value, bd->cfg->erase_size); // mirror to disk file? if (bd->disk) { off_t res1 = lseek(bd->disk->fd, - (off_t)block*cfg->block_size, + (off_t)block*bd->cfg->erase_size, SEEK_SET); if (res1 < 0) { int err = -errno; @@ -420,7 +403,7 @@ int lfs_emubd_erase(const struct lfs_config *cfg, lfs_block_t block) { ssize_t res2 = write(bd->disk->fd, bd->disk->scratch, - cfg->block_size); + bd->cfg->erase_size); if (res2 < 0) { int err = -errno; LFS_EMUBD_TRACE("lfs_emubd_erase -> %d", err); @@ -430,7 +413,7 @@ int lfs_emubd_erase(const struct lfs_config *cfg, lfs_block_t block) { } // track erases - bd->erased += cfg->block_size; + bd->erased += bd->cfg->erase_size; if (bd->cfg->erase_sleep) { int err = nanosleep(&(struct timespec){ .tv_sec=bd->cfg->erase_sleep/1000000000, @@ -573,7 +556,7 @@ lfs_emubd_swear_t lfs_emubd_wear(const struct lfs_config *cfg, lfs_emubd_t *bd = cfg->context; // check if block is valid - LFS_ASSERT(block < cfg->block_count); + LFS_ASSERT(block < bd->cfg->erase_count); // get the wear lfs_emubd_wear_t wear; @@ -594,7 +577,7 @@ int lfs_emubd_setwear(const struct lfs_config *cfg, lfs_emubd_t *bd = cfg->context; // check if block is valid - LFS_ASSERT(block < cfg->block_count); + LFS_ASSERT(block < bd->cfg->erase_count); // set the wear lfs_emubd_block_t *b = lfs_emubd_mutblock(cfg, &bd->blocks[block]); @@ -634,13 +617,13 @@ int lfs_emubd_copy(const struct lfs_config *cfg, lfs_emubd_t *copy) { lfs_emubd_t *bd = cfg->context; // lazily copy over our block array - copy->blocks = malloc(cfg->block_count * sizeof(lfs_emubd_block_t*)); + copy->blocks = malloc(bd->cfg->erase_count * sizeof(lfs_emubd_block_t*)); if (!copy->blocks) { LFS_EMUBD_TRACE("lfs_emubd_copy -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } - for (size_t i = 0; i < cfg->block_count; i++) { + for (size_t i = 0; i < bd->cfg->erase_count; i++) { copy->blocks[i] = lfs_emubd_incblock(bd->blocks[i]); } diff --git a/bd/lfs_emubd.h b/bd/lfs_emubd.h index 35a411fe..9049649f 100644 --- a/bd/lfs_emubd.h +++ b/bd/lfs_emubd.h @@ -67,6 +67,18 @@ typedef int64_t lfs_emubd_ssleep_t; // emubd config, this is required for testing struct lfs_emubd_config { + // Minimum size of a read operation in bytes. + lfs_size_t read_size; + + // Minimum size of a program operation in bytes. + lfs_size_t prog_size; + + // Size of an erase operation in bytes. + lfs_size_t erase_size; + + // Number of erase blocks on the device. + lfs_size_t erase_count; + // 8-bit erase value to use for simulating erases. -1 does not simulate // erases, which can speed up testing by avoiding the extra block-device // operations to store the erase value. @@ -149,11 +161,7 @@ typedef struct lfs_emubd { /// Block device API /// // Create an emulating block device using the geometry in lfs_config -// -// Note that filebd is used if a path is provided, if path is NULL -// emubd will use rambd which can be much faster. -int lfs_emubd_create(const struct lfs_config *cfg, const char *path); -int lfs_emubd_createcfg(const struct lfs_config *cfg, const char *path, +int lfs_emubd_create(const struct lfs_config *cfg, const struct lfs_emubd_config *bdcfg); // Clean up memory associated with block device diff --git a/bd/lfs_filebd.c b/bd/lfs_filebd.c index 780c8f90..4ff25d44 100644 --- a/bd/lfs_filebd.c +++ b/bd/lfs_filebd.c @@ -15,18 +15,22 @@ #include #endif -int lfs_filebd_create(const struct lfs_config *cfg, const char *path) { +int lfs_filebd_create(const struct lfs_config *cfg, const char *path, + const struct lfs_filebd_config *bdcfg) { LFS_FILEBD_TRACE("lfs_filebd_create(%p {.context=%p, " - ".read=%p, .prog=%p, .erase=%p, .sync=%p, " - ".read_size=%"PRIu32", .prog_size=%"PRIu32", " - ".block_size=%"PRIu32", .block_count=%"PRIu32"}, " - "\"%s\")", + ".read=%p, .prog=%p, .erase=%p, .sync=%p}, " + "\"%s\", " + "%p {.read_size=%"PRIu32", .prog_size=%"PRIu32", " + ".erase_size=%"PRIu32", .erase_count=%"PRIu32"})", (void*)cfg, cfg->context, (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, - cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, - path, (void*)bdcfg, bdcfg->erase_value); + path, + (void*)bdcfg, + bdcfg->read_size, bdcfg->prog_size, bdcfg->erase_size, + bdcfg->erase_count); lfs_filebd_t *bd = cfg->context; + bd->cfg = bdcfg; // open file #ifdef _WIN32 @@ -66,17 +70,17 @@ int lfs_filebd_read(const struct lfs_config *cfg, lfs_block_t block, lfs_filebd_t *bd = cfg->context; // check if read is valid - LFS_ASSERT(block < cfg->block_count); - LFS_ASSERT(off % cfg->read_size == 0); - LFS_ASSERT(size % cfg->read_size == 0); - LFS_ASSERT(off+size <= cfg->block_size); + LFS_ASSERT(block < bd->cfg->erase_count); + LFS_ASSERT(off % bd->cfg->read_size == 0); + LFS_ASSERT(size % bd->cfg->read_size == 0); + LFS_ASSERT(off+size <= bd->cfg->erase_size); // zero for reproducibility (in case file is truncated) memset(buffer, 0, size); // read off_t res1 = lseek(bd->fd, - (off_t)block*cfg->block_size + (off_t)off, SEEK_SET); + (off_t)block*bd->cfg->erase_size + (off_t)off, SEEK_SET); if (res1 < 0) { int err = -errno; LFS_FILEBD_TRACE("lfs_filebd_read -> %d", err); @@ -102,14 +106,14 @@ int lfs_filebd_prog(const struct lfs_config *cfg, lfs_block_t block, lfs_filebd_t *bd = cfg->context; // check if write is valid - LFS_ASSERT(block < cfg->block_count); - LFS_ASSERT(off % cfg->prog_size == 0); - LFS_ASSERT(size % cfg->prog_size == 0); - LFS_ASSERT(off+size <= cfg->block_size); + LFS_ASSERT(block < bd->cfg->erase_count); + LFS_ASSERT(off % bd->cfg->prog_size == 0); + LFS_ASSERT(size % bd->cfg->prog_size == 0); + LFS_ASSERT(off+size <= bd->cfg->erase_size); // program data off_t res1 = lseek(bd->fd, - (off_t)block*cfg->block_size + (off_t)off, SEEK_SET); + (off_t)block*bd->cfg->erase_size + (off_t)off, SEEK_SET); if (res1 < 0) { int err = -errno; LFS_FILEBD_TRACE("lfs_filebd_prog -> %d", err); @@ -129,10 +133,11 @@ int lfs_filebd_prog(const struct lfs_config *cfg, lfs_block_t block, int lfs_filebd_erase(const struct lfs_config *cfg, lfs_block_t block) { LFS_FILEBD_TRACE("lfs_filebd_erase(%p, 0x%"PRIx32" (%"PRIu32"))", - (void*)cfg, block, cfg->block_size); + (void*)cfg, block, ((lfs_file_t*)cfg->context)->cfg->erase_size); + lfs_filebd_t *bd = cfg->context; // check if erase is valid - LFS_ASSERT(block < cfg->block_count); + LFS_ASSERT(block < bd->cfg->erase_count); // erase is a noop (void)block; diff --git a/bd/lfs_filebd.h b/bd/lfs_filebd.h index 0f24996a..d7d2fd95 100644 --- a/bd/lfs_filebd.h +++ b/bd/lfs_filebd.h @@ -26,14 +26,31 @@ extern "C" #endif #endif +// filebd config +struct lfs_filebd_config { + // Minimum size of a read operation in bytes. + lfs_size_t read_size; + + // Minimum size of a program operation in bytes. + lfs_size_t prog_size; + + // Size of an erase operation in bytes. + lfs_size_t erase_size; + + // Number of erase blocks on the device. + lfs_size_t erase_count; +}; + // filebd state typedef struct lfs_filebd { int fd; + const struct lfs_filebd_config *cfg; } lfs_filebd_t; -// Create a file block device using the geometry in lfs_config -int lfs_filebd_create(const struct lfs_config *cfg, const char *path); +// Create a file block device +int lfs_filebd_create(const struct lfs_config *cfg, const char *path, + const struct lfs_filebd_config *bdcfg); // Clean up memory associated with block device int lfs_filebd_destroy(const struct lfs_config *cfg); diff --git a/bd/lfs_rambd.c b/bd/lfs_rambd.c index ab180b93..a6a05727 100644 --- a/bd/lfs_rambd.c +++ b/bd/lfs_rambd.c @@ -7,18 +7,19 @@ */ #include "bd/lfs_rambd.h" -int lfs_rambd_createcfg(const struct lfs_config *cfg, +int lfs_rambd_create(const struct lfs_config *cfg, const struct lfs_rambd_config *bdcfg) { - LFS_RAMBD_TRACE("lfs_rambd_createcfg(%p {.context=%p, " - ".read=%p, .prog=%p, .erase=%p, .sync=%p, " - ".read_size=%"PRIu32", .prog_size=%"PRIu32", " - ".block_size=%"PRIu32", .block_count=%"PRIu32"}, " - "%p {.buffer=%p})", + LFS_RAMBD_TRACE("lfs_rambd_create(%p {.context=%p, " + ".read=%p, .prog=%p, .erase=%p, .sync=%p}, " + "%p {.read_size=%"PRIu32", .prog_size=%"PRIu32", " + ".erase_size=%"PRIu32", .erase_count=%"PRIu32", " + ".buffer=%p})", (void*)cfg, cfg->context, (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, - cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count, - (void*)bdcfg, bdcfg->buffer); + (void*)bdcfg, + bdcfg->read_size, bdcfg->prog_size, bdcfg->erase_size, + bdcfg->erase_count, bdcfg->buffer); lfs_rambd_t *bd = cfg->context; bd->cfg = bdcfg; @@ -26,35 +27,20 @@ int lfs_rambd_createcfg(const struct lfs_config *cfg, if (bd->cfg->buffer) { bd->buffer = bd->cfg->buffer; } else { - bd->buffer = lfs_malloc(cfg->block_size * cfg->block_count); + bd->buffer = lfs_malloc(bd->cfg->erase_size * bd->cfg->erase_count); if (!bd->buffer) { - LFS_RAMBD_TRACE("lfs_rambd_createcfg -> %d", LFS_ERR_NOMEM); + LFS_RAMBD_TRACE("lfs_rambd_create -> %d", LFS_ERR_NOMEM); return LFS_ERR_NOMEM; } } // zero for reproducibility - memset(bd->buffer, 0, cfg->block_size * cfg->block_count); + memset(bd->buffer, 0, bd->cfg->erase_size * bd->cfg->erase_count); - LFS_RAMBD_TRACE("lfs_rambd_createcfg -> %d", 0); + LFS_RAMBD_TRACE("lfs_rambd_create -> %d", 0); return 0; } -int lfs_rambd_create(const struct lfs_config *cfg) { - LFS_RAMBD_TRACE("lfs_rambd_create(%p {.context=%p, " - ".read=%p, .prog=%p, .erase=%p, .sync=%p, " - ".read_size=%"PRIu32", .prog_size=%"PRIu32", " - ".block_size=%"PRIu32", .block_count=%"PRIu32"})", - (void*)cfg, cfg->context, - (void*)(uintptr_t)cfg->read, (void*)(uintptr_t)cfg->prog, - (void*)(uintptr_t)cfg->erase, (void*)(uintptr_t)cfg->sync, - cfg->read_size, cfg->prog_size, cfg->block_size, cfg->block_count); - static const struct lfs_rambd_config defaults = {0}; - int err = lfs_rambd_createcfg(cfg, &defaults); - LFS_RAMBD_TRACE("lfs_rambd_create -> %d", err); - return err; -} - int lfs_rambd_destroy(const struct lfs_config *cfg) { LFS_RAMBD_TRACE("lfs_rambd_destroy(%p)", (void*)cfg); // clean up memory @@ -74,13 +60,13 @@ int lfs_rambd_read(const struct lfs_config *cfg, lfs_block_t block, lfs_rambd_t *bd = cfg->context; // check if read is valid - LFS_ASSERT(block < cfg->block_count); - LFS_ASSERT(off % cfg->read_size == 0); - LFS_ASSERT(size % cfg->read_size == 0); - LFS_ASSERT(off+size <= cfg->block_size); + LFS_ASSERT(block < bd->cfg->erase_count); + LFS_ASSERT(off % bd->cfg->read_size == 0); + LFS_ASSERT(size % bd->cfg->read_size == 0); + LFS_ASSERT(off+size <= bd->cfg->erase_size); // read data - memcpy(buffer, &bd->buffer[block*cfg->block_size + off], size); + memcpy(buffer, &bd->buffer[block*bd->cfg->erase_size + off], size); LFS_RAMBD_TRACE("lfs_rambd_read -> %d", 0); return 0; @@ -94,13 +80,13 @@ int lfs_rambd_prog(const struct lfs_config *cfg, lfs_block_t block, lfs_rambd_t *bd = cfg->context; // check if write is valid - LFS_ASSERT(block < cfg->block_count); - LFS_ASSERT(off % cfg->prog_size == 0); - LFS_ASSERT(size % cfg->prog_size == 0); - LFS_ASSERT(off+size <= cfg->block_size); + LFS_ASSERT(block < bd->cfg->erase_count); + LFS_ASSERT(off % bd->cfg->prog_size == 0); + LFS_ASSERT(size % bd->cfg->prog_size == 0); + LFS_ASSERT(off+size <= bd->cfg->erase_size); // program data - memcpy(&bd->buffer[block*cfg->block_size + off], buffer, size); + memcpy(&bd->buffer[block*bd->cfg->erase_size + off], buffer, size); LFS_RAMBD_TRACE("lfs_rambd_prog -> %d", 0); return 0; @@ -108,10 +94,11 @@ int lfs_rambd_prog(const struct lfs_config *cfg, lfs_block_t block, int lfs_rambd_erase(const struct lfs_config *cfg, lfs_block_t block) { LFS_RAMBD_TRACE("lfs_rambd_erase(%p, 0x%"PRIx32" (%"PRIu32"))", - (void*)cfg, block, cfg->block_size); + (void*)cfg, block, ((lfs_rambd_t*)cfg->context)->cfg->erase_size); + lfs_rambd_t *bd = cfg->context; // check if erase is valid - LFS_ASSERT(block < cfg->block_count); + LFS_ASSERT(block < bd->cfg->erase_count); // erase is a noop (void)block; diff --git a/bd/lfs_rambd.h b/bd/lfs_rambd.h index 34246802..86637026 100644 --- a/bd/lfs_rambd.h +++ b/bd/lfs_rambd.h @@ -26,8 +26,20 @@ extern "C" #endif #endif -// rambd config (optional) +// rambd config struct lfs_rambd_config { + // Minimum size of a read operation in bytes. + lfs_size_t read_size; + + // Minimum size of a program operation in bytes. + lfs_size_t prog_size; + + // Size of an erase operation in bytes. + lfs_size_t erase_size; + + // Number of erase blocks on the device. + lfs_size_t erase_count; + // Optional statically allocated buffer for the block device. void *buffer; }; @@ -39,9 +51,8 @@ typedef struct lfs_rambd { } lfs_rambd_t; -// Create a RAM block device using the geometry in lfs_config -int lfs_rambd_create(const struct lfs_config *cfg); -int lfs_rambd_createcfg(const struct lfs_config *cfg, +// Create a RAM block device +int lfs_rambd_create(const struct lfs_config *cfg, const struct lfs_rambd_config *bdcfg); // Clean up memory associated with block device diff --git a/lfs.h b/lfs.h index ae609e43..d919aabc 100644 --- a/lfs.h +++ b/lfs.h @@ -197,9 +197,8 @@ struct lfs_config { // will be a multiple of this value. lfs_size_t prog_size; - // Minimum size of an erase operation in bytes. All erase operations - // will be a multiple of this value. This must be a multiple of the read - // and program sizes. + // Size of an erase operation in bytes. This must be a multiple of the + // read and program sizes. // // If zero, the block_size is used as the erase_size. This is mostly for // backwards compatibility. diff --git a/runners/bench_runner.c b/runners/bench_runner.c index ba791b25..32cf181f 100644 --- a/runners/bench_runner.c +++ b/runners/bench_runner.c @@ -1271,9 +1271,9 @@ static void list_geometries(void) { builtin_geometries[g].name, READ_SIZE, PROG_SIZE, - BLOCK_SIZE, - BLOCK_COUNT, - BLOCK_SIZE*BLOCK_COUNT); + ERASE_SIZE, + ERASE_COUNT, + ERASE_SIZE*ERASE_COUNT); } } @@ -1316,6 +1316,7 @@ void perm_run( .sync = lfs_emubd_sync, .read_size = READ_SIZE, .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, .block_size = BLOCK_SIZE, .block_count = BLOCK_COUNT, .block_cycles = BLOCK_CYCLES, @@ -1324,6 +1325,10 @@ void perm_run( }; struct lfs_emubd_config bdcfg = { + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, + .erase_count = ERASE_COUNT, .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, @@ -1333,7 +1338,7 @@ void perm_run( .erase_sleep = bench_erase_sleep, }; - int err = lfs_emubd_createcfg(&cfg, bench_disk_path, &bdcfg); + int err = lfs_emubd_create(&cfg, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -1761,19 +1766,19 @@ int main(int argc, char **argv) { = BENCH_LIT(sizes[0]); geometry->defines[PROG_SIZE_i] = BENCH_LIT(sizes[1]); - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = BENCH_LIT(sizes[2]); } else if (count >= 2) { geometry->defines[PROG_SIZE_i] = BENCH_LIT(sizes[0]); - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = BENCH_LIT(sizes[1]); } else { - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = BENCH_LIT(sizes[0]); } if (count >= 4) { - geometry->defines[BLOCK_COUNT_i] + geometry->defines[ERASE_COUNT_i] = BENCH_LIT(sizes[3]); } optarg = s; @@ -1805,19 +1810,19 @@ int main(int argc, char **argv) { = BENCH_LIT(sizes[0]); geometry->defines[PROG_SIZE_i] = BENCH_LIT(sizes[1]); - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = BENCH_LIT(sizes[2]); } else if (count >= 2) { geometry->defines[PROG_SIZE_i] = BENCH_LIT(sizes[0]); - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = BENCH_LIT(sizes[1]); } else { - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = BENCH_LIT(sizes[0]); } if (count >= 4) { - geometry->defines[BLOCK_COUNT_i] + geometry->defines[ERASE_COUNT_i] = BENCH_LIT(sizes[3]); } optarg = s; diff --git a/runners/bench_runner.h b/runners/bench_runner.h index 6296c091..b072970e 100644 --- a/runners/bench_runner.h +++ b/runners/bench_runner.h @@ -89,18 +89,22 @@ intmax_t bench_define(size_t define); #define READ_SIZE_i 0 #define PROG_SIZE_i 1 -#define BLOCK_SIZE_i 2 -#define BLOCK_COUNT_i 3 -#define CACHE_SIZE_i 4 -#define LOOKAHEAD_SIZE_i 5 -#define BLOCK_CYCLES_i 6 -#define ERASE_VALUE_i 7 -#define ERASE_CYCLES_i 8 -#define BADBLOCK_BEHAVIOR_i 9 -#define POWERLOSS_BEHAVIOR_i 10 +#define ERASE_SIZE_i 2 +#define ERASE_COUNT_i 3 +#define BLOCK_SIZE_i 4 +#define BLOCK_COUNT_i 5 +#define CACHE_SIZE_i 6 +#define LOOKAHEAD_SIZE_i 7 +#define BLOCK_CYCLES_i 8 +#define ERASE_VALUE_i 9 +#define ERASE_CYCLES_i 10 +#define BADBLOCK_BEHAVIOR_i 11 +#define POWERLOSS_BEHAVIOR_i 12 #define READ_SIZE bench_define(READ_SIZE_i) #define PROG_SIZE bench_define(PROG_SIZE_i) +#define ERASE_SIZE bench_define(ERASE_SIZE_i) +#define ERASE_COUNT bench_define(ERASE_COUNT_i) #define BLOCK_SIZE bench_define(BLOCK_SIZE_i) #define BLOCK_COUNT bench_define(BLOCK_COUNT_i) #define CACHE_SIZE bench_define(CACHE_SIZE_i) @@ -113,9 +117,11 @@ intmax_t bench_define(size_t define); #define BENCH_IMPLICIT_DEFINES \ BENCH_DEF(READ_SIZE, PROG_SIZE) \ - BENCH_DEF(PROG_SIZE, BLOCK_SIZE) \ - BENCH_DEF(BLOCK_SIZE, 0) \ - BENCH_DEF(BLOCK_COUNT, (1024*1024)/BLOCK_SIZE) \ + BENCH_DEF(PROG_SIZE, ERASE_SIZE) \ + BENCH_DEF(ERASE_SIZE, 0) \ + BENCH_DEF(ERASE_COUNT, (1024*1024)/BLOCK_SIZE) \ + BENCH_DEF(BLOCK_SIZE, ERASE_SIZE) \ + BENCH_DEF(BLOCK_COUNT, ERASE_COUNT/lfs_max(BLOCK_SIZE/ERASE_SIZE,1))\ BENCH_DEF(CACHE_SIZE, lfs_max(64,lfs_max(READ_SIZE,PROG_SIZE))) \ BENCH_DEF(LOOKAHEAD_SIZE, 16) \ BENCH_DEF(BLOCK_CYCLES, -1) \ @@ -125,7 +131,7 @@ intmax_t bench_define(size_t define); BENCH_DEF(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP) #define BENCH_GEOMETRY_DEFINE_COUNT 4 -#define BENCH_IMPLICIT_DEFINE_COUNT 11 +#define BENCH_IMPLICIT_DEFINE_COUNT 13 #endif diff --git a/runners/test_runner.c b/runners/test_runner.c index abc867c2..af6bd793 100644 --- a/runners/test_runner.c +++ b/runners/test_runner.c @@ -1312,9 +1312,9 @@ static void list_geometries(void) { builtin_geometries[g].name, READ_SIZE, PROG_SIZE, - BLOCK_SIZE, - BLOCK_COUNT, - BLOCK_SIZE*BLOCK_COUNT); + ERASE_SIZE, + ERASE_COUNT, + ERASE_SIZE*ERASE_COUNT); } } @@ -1341,6 +1341,7 @@ static void run_powerloss_none( .sync = lfs_emubd_sync, .read_size = READ_SIZE, .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, .block_size = BLOCK_SIZE, .block_count = BLOCK_COUNT, .block_cycles = BLOCK_CYCLES, @@ -1349,6 +1350,10 @@ static void run_powerloss_none( }; struct lfs_emubd_config bdcfg = { + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, + .erase_count = ERASE_COUNT, .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, @@ -1358,7 +1363,7 @@ static void run_powerloss_none( .erase_sleep = test_erase_sleep, }; - int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg); + int err = lfs_emubd_create(&cfg, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -1410,6 +1415,7 @@ static void run_powerloss_linear( .sync = lfs_emubd_sync, .read_size = READ_SIZE, .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, .block_size = BLOCK_SIZE, .block_count = BLOCK_COUNT, .block_cycles = BLOCK_CYCLES, @@ -1418,6 +1424,10 @@ static void run_powerloss_linear( }; struct lfs_emubd_config bdcfg = { + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, + .erase_count = ERASE_COUNT, .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, @@ -1431,7 +1441,7 @@ static void run_powerloss_linear( .powerloss_data = &powerloss_jmp, }; - int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg); + int err = lfs_emubd_create(&cfg, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -1496,6 +1506,7 @@ static void run_powerloss_log( .sync = lfs_emubd_sync, .read_size = READ_SIZE, .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, .block_size = BLOCK_SIZE, .block_count = BLOCK_COUNT, .block_cycles = BLOCK_CYCLES, @@ -1504,6 +1515,10 @@ static void run_powerloss_log( }; struct lfs_emubd_config bdcfg = { + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, + .erase_count = ERASE_COUNT, .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, @@ -1517,7 +1532,7 @@ static void run_powerloss_log( .powerloss_data = &powerloss_jmp, }; - int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg); + int err = lfs_emubd_create(&cfg, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -1580,6 +1595,7 @@ static void run_powerloss_cycles( .sync = lfs_emubd_sync, .read_size = READ_SIZE, .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, .block_size = BLOCK_SIZE, .block_count = BLOCK_COUNT, .block_cycles = BLOCK_CYCLES, @@ -1588,6 +1604,10 @@ static void run_powerloss_cycles( }; struct lfs_emubd_config bdcfg = { + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, + .erase_count = ERASE_COUNT, .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, @@ -1601,7 +1621,7 @@ static void run_powerloss_cycles( .powerloss_data = &powerloss_jmp, }; - int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg); + int err = lfs_emubd_create(&cfg, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -1762,6 +1782,7 @@ static void run_powerloss_exhaustive( .sync = lfs_emubd_sync, .read_size = READ_SIZE, .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, .block_size = BLOCK_SIZE, .block_count = BLOCK_COUNT, .block_cycles = BLOCK_CYCLES, @@ -1770,6 +1791,10 @@ static void run_powerloss_exhaustive( }; struct lfs_emubd_config bdcfg = { + .read_size = READ_SIZE, + .prog_size = PROG_SIZE, + .erase_size = ERASE_SIZE, + .erase_count = ERASE_COUNT, .erase_value = ERASE_VALUE, .erase_cycles = ERASE_CYCLES, .badblock_behavior = BADBLOCK_BEHAVIOR, @@ -1782,7 +1807,7 @@ static void run_powerloss_exhaustive( .powerloss_data = NULL, }; - int err = lfs_emubd_createcfg(&cfg, test_disk_path, &bdcfg); + int err = lfs_emubd_create(&cfg, &bdcfg); if (err) { fprintf(stderr, "error: could not create block device: %d\n", err); exit(-1); @@ -2299,19 +2324,19 @@ int main(int argc, char **argv) { = TEST_LIT(sizes[0]); geometry->defines[PROG_SIZE_i] = TEST_LIT(sizes[1]); - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = TEST_LIT(sizes[2]); } else if (count >= 2) { geometry->defines[PROG_SIZE_i] = TEST_LIT(sizes[0]); - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = TEST_LIT(sizes[1]); } else { - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = TEST_LIT(sizes[0]); } if (count >= 4) { - geometry->defines[BLOCK_COUNT_i] + geometry->defines[ERASE_COUNT_i] = TEST_LIT(sizes[3]); } optarg = s; @@ -2343,19 +2368,19 @@ int main(int argc, char **argv) { = TEST_LIT(sizes[0]); geometry->defines[PROG_SIZE_i] = TEST_LIT(sizes[1]); - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = TEST_LIT(sizes[2]); } else if (count >= 2) { geometry->defines[PROG_SIZE_i] = TEST_LIT(sizes[0]); - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = TEST_LIT(sizes[1]); } else { - geometry->defines[BLOCK_SIZE_i] + geometry->defines[ERASE_SIZE_i] = TEST_LIT(sizes[0]); } if (count >= 4) { - geometry->defines[BLOCK_COUNT_i] + geometry->defines[ERASE_COUNT_i] = TEST_LIT(sizes[3]); } optarg = s; diff --git a/runners/test_runner.h b/runners/test_runner.h index 9ff1f790..b1217a43 100644 --- a/runners/test_runner.h +++ b/runners/test_runner.h @@ -82,18 +82,22 @@ intmax_t test_define(size_t define); #define READ_SIZE_i 0 #define PROG_SIZE_i 1 -#define BLOCK_SIZE_i 2 -#define BLOCK_COUNT_i 3 -#define CACHE_SIZE_i 4 -#define LOOKAHEAD_SIZE_i 5 -#define BLOCK_CYCLES_i 6 -#define ERASE_VALUE_i 7 -#define ERASE_CYCLES_i 8 -#define BADBLOCK_BEHAVIOR_i 9 -#define POWERLOSS_BEHAVIOR_i 10 +#define ERASE_SIZE_i 2 +#define ERASE_COUNT_i 3 +#define BLOCK_SIZE_i 4 +#define BLOCK_COUNT_i 5 +#define CACHE_SIZE_i 6 +#define LOOKAHEAD_SIZE_i 7 +#define BLOCK_CYCLES_i 8 +#define ERASE_VALUE_i 9 +#define ERASE_CYCLES_i 10 +#define BADBLOCK_BEHAVIOR_i 11 +#define POWERLOSS_BEHAVIOR_i 12 #define READ_SIZE TEST_DEFINE(READ_SIZE_i) #define PROG_SIZE TEST_DEFINE(PROG_SIZE_i) +#define ERASE_SIZE TEST_DEFINE(ERASE_SIZE_i) +#define ERASE_COUNT TEST_DEFINE(ERASE_COUNT_i) #define BLOCK_SIZE TEST_DEFINE(BLOCK_SIZE_i) #define BLOCK_COUNT TEST_DEFINE(BLOCK_COUNT_i) #define CACHE_SIZE TEST_DEFINE(CACHE_SIZE_i) @@ -106,9 +110,11 @@ intmax_t test_define(size_t define); #define TEST_IMPLICIT_DEFINES \ TEST_DEF(READ_SIZE, PROG_SIZE) \ - TEST_DEF(PROG_SIZE, BLOCK_SIZE) \ - TEST_DEF(BLOCK_SIZE, 0) \ - TEST_DEF(BLOCK_COUNT, (1024*1024)/BLOCK_SIZE) \ + TEST_DEF(PROG_SIZE, ERASE_SIZE) \ + TEST_DEF(ERASE_SIZE, 0) \ + TEST_DEF(ERASE_COUNT, (1024*1024)/ERASE_SIZE) \ + TEST_DEF(BLOCK_SIZE, ERASE_SIZE) \ + TEST_DEF(BLOCK_COUNT, ERASE_COUNT/lfs_max(BLOCK_SIZE/ERASE_SIZE,1)) \ TEST_DEF(CACHE_SIZE, lfs_max(64,lfs_max(READ_SIZE,PROG_SIZE))) \ TEST_DEF(LOOKAHEAD_SIZE, 16) \ TEST_DEF(BLOCK_CYCLES, -1) \ @@ -117,8 +123,8 @@ intmax_t test_define(size_t define); TEST_DEF(BADBLOCK_BEHAVIOR, LFS_EMUBD_BADBLOCK_PROGERROR) \ TEST_DEF(POWERLOSS_BEHAVIOR, LFS_EMUBD_POWERLOSS_NOOP) -#define TEST_IMPLICIT_DEFINE_COUNT 11 #define TEST_GEOMETRY_DEFINE_COUNT 4 +#define TEST_IMPLICIT_DEFINE_COUNT 13 #endif diff --git a/tests/test_alloc.toml b/tests/test_alloc.toml index 205efbb1..916fc2a5 100644 --- a/tests/test_alloc.toml +++ b/tests/test_alloc.toml @@ -460,8 +460,8 @@ code = ''' # chained dir exhaustion test [cases.test_alloc_chained_dir_exhaustion] -if = 'BLOCK_SIZE == 512' -defines.BLOCK_COUNT = 1024 +if = 'ERASE_SIZE == 512' +defines.ERASE_COUNT = 1024 code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -538,8 +538,8 @@ code = ''' # split dir test [cases.test_alloc_split_dir] -if = 'BLOCK_SIZE == 512' -defines.BLOCK_COUNT = 1024 +if = 'ERASE_SIZE == 512' +defines.ERASE_COUNT = 1024 code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -587,8 +587,8 @@ code = ''' # outdated lookahead test [cases.test_alloc_outdated_lookahead] -if = 'BLOCK_SIZE == 512' -defines.BLOCK_COUNT = 1024 +if = 'ERASE_SIZE == 512' +defines.ERASE_COUNT = 1024 code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; @@ -655,8 +655,8 @@ code = ''' # outdated lookahead and split dir test [cases.test_alloc_outdated_lookahead_split_dir] -if = 'BLOCK_SIZE == 512' -defines.BLOCK_COUNT = 1024 +if = 'ERASE_SIZE == 512' +defines.ERASE_COUNT = 1024 code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; diff --git a/tests/test_badblocks.toml b/tests/test_badblocks.toml index 5035b931..6502f694 100644 --- a/tests/test_badblocks.toml +++ b/tests/test_badblocks.toml @@ -2,7 +2,7 @@ if = '(int32_t)BLOCK_CYCLES == -1' [cases.test_badblocks_single] -defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.ERASE_COUNT = 256 # small bd so test runs faster defines.ERASE_CYCLES = 0xffffffff defines.ERASE_VALUE = [0x00, 0xff, -1] defines.BADBLOCK_BEHAVIOR = [ @@ -82,7 +82,7 @@ code = ''' ''' [cases.test_badblocks_region_corruption] # (causes cascading failures) -defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.ERASE_COUNT = 256 # small bd so test runs faster defines.ERASE_CYCLES = 0xffffffff defines.ERASE_VALUE = [0x00, 0xff, -1] defines.BADBLOCK_BEHAVIOR = [ @@ -161,7 +161,7 @@ code = ''' ''' [cases.test_badblocks_alternating_corruption] # (causes cascading failures) -defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.ERASE_COUNT = 256 # small bd so test runs faster defines.ERASE_CYCLES = 0xffffffff defines.ERASE_VALUE = [0x00, 0xff, -1] defines.BADBLOCK_BEHAVIOR = [ diff --git a/tests/test_exhaustion.toml b/tests/test_exhaustion.toml index 2cf6aed1..273dd98c 100644 --- a/tests/test_exhaustion.toml +++ b/tests/test_exhaustion.toml @@ -1,7 +1,7 @@ # test running a filesystem to exhaustion [cases.test_exhaustion_normal] defines.ERASE_CYCLES = 10 -defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.ERASE_COUNT = 256 # small bd so test runs faster defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2' defines.BADBLOCK_BEHAVIOR = [ 'LFS_EMUBD_BADBLOCK_PROGERROR', @@ -94,7 +94,7 @@ exhausted: # which also requires expanding superblocks [cases.test_exhaustion_superblocks] defines.ERASE_CYCLES = 10 -defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.ERASE_COUNT = 256 # small bd so test runs faster defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2' defines.BADBLOCK_BEHAVIOR = [ 'LFS_EMUBD_BADBLOCK_PROGERROR', @@ -188,7 +188,7 @@ exhausted: # wear-level test running a filesystem to exhaustion [cases.test_exhuastion_wear_leveling] defines.ERASE_CYCLES = 20 -defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.ERASE_COUNT = 256 # small bd so test runs faster defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2' defines.FILES = 10 code = ''' @@ -288,7 +288,7 @@ exhausted: # wear-level test + expanding superblock [cases.test_exhaustion_wear_leveling_superblocks] defines.ERASE_CYCLES = 20 -defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.ERASE_COUNT = 256 # small bd so test runs faster defines.BLOCK_CYCLES = 'ERASE_CYCLES / 2' defines.FILES = 10 code = ''' @@ -385,7 +385,7 @@ exhausted: # test that we wear blocks roughly evenly [cases.test_exhaustion_wear_distribution] defines.ERASE_CYCLES = 0xffffffff -defines.BLOCK_COUNT = 256 # small bd so test runs faster +defines.ERASE_COUNT = 256 # small bd so test runs faster defines.BLOCK_CYCLES = [5, 4, 3, 2, 1] defines.CYCLES = 100 defines.FILES = 10 diff --git a/tests/test_superblocks.toml b/tests/test_superblocks.toml index 60bda767..2625ecc3 100644 --- a/tests/test_superblocks.toml +++ b/tests/test_superblocks.toml @@ -149,3 +149,51 @@ code = ''' assert(info.type == LFS_TYPE_REG); lfs_unmount(&lfs) => 0; ''' + +# mount with unknown block_size +[cases.test_superblocks_unknown_block_size] +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + + cfg->block_size = 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; + lfs_file_open(&lfs, &file, "test", + LFS_O_CREAT | LFS_O_EXCL | LFS_O_WRONLY) => 0; + lfs_file_write(&lfs, &file, "hello!", 6) => 6; + lfs_file_close(&lfs, &file) => 0; + lfs_unmount(&lfs) => 0; + + lfs_mount(&lfs, cfg) => 0; + lfs_file_open(&lfs, &file, "test", LFS_O_RDONLY) => 0; + uint8_t buffer[256]; + lfs_file_read(&lfs, &file, buffer, sizeof(buffer)) => 6; + lfs_file_close(&lfs, &file) => 0; + assert(memcmp(buffer, "hello!", 6) == 0); + lfs_unmount(&lfs) => 0; +''' + +# mount with unknown block_count +[cases.test_superblocks_unknown_block_count] +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + + cfg->block_count = 0; + lfs_mount(&lfs, cfg) => 0; + lfs_file_t file; + lfs_file_open(&lfs, &file, "test", + LFS_O_CREAT | LFS_O_EXCL | LFS_O_WRONLY) => 0; + lfs_file_write(&lfs, &file, "hello!", 6) => 6; + lfs_file_close(&lfs, &file) => 0; + lfs_unmount(&lfs) => 0; + + lfs_mount(&lfs, cfg) => 0; + lfs_file_open(&lfs, &file, "test", LFS_O_RDONLY) => 0; + uint8_t buffer[256]; + lfs_file_read(&lfs, &file, buffer, sizeof(buffer)) => 6; + lfs_file_close(&lfs, &file) => 0; + assert(memcmp(buffer, "hello!", 6) == 0); + lfs_unmount(&lfs) => 0; +''' From c8bc6f96cda13e78d502fae1ce212328fb921c77 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Fri, 11 Nov 2022 15:34:12 -0600 Subject: [PATCH 5/9] Added lfs_fs_stat for accessing configuration in the superblocks This is necessary for accessing format-specific information stored in the superblock after mounting. As is common in other filesystems this also provides the filesystem usage. However collecting the filesystem usage is more expensive in littlefs than other filesystem (and less accurate because of CoW), so this may need to be cached after mount in the future. --- lfs.c | 29 +++++++++++++++++++ lfs.h | 32 +++++++++++++++++++++ tests/test_superblocks.toml | 55 +++++++++++++++++++++++++++++++++++++ 3 files changed, 116 insertions(+) diff --git a/lfs.c b/lfs.c index cf44f95f..38d09273 100644 --- a/lfs.c +++ b/lfs.c @@ -4825,6 +4825,21 @@ static lfs_ssize_t lfs_fs_rawsize(lfs_t *lfs) { return size; } +static int lfs_fs_rawstat(lfs_t *lfs, struct lfs_fsinfo *fsinfo) { + lfs_ssize_t usage = lfs_fs_rawsize(lfs); + if (usage < 0) { + return usage; + } + + fsinfo->block_size = lfs->block_size; + fsinfo->block_count = lfs->block_count; + fsinfo->block_usage = usage; + fsinfo->name_max = lfs->name_max; + fsinfo->file_max = lfs->file_max; + fsinfo->attr_max = lfs->attr_max; + return 0; +} + #ifdef LFS_MIGRATE ////// Migration from littelfs v1 below this ////// @@ -5948,6 +5963,20 @@ int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir) { return err; } +int lfs_fs_stat(lfs_t *lfs, struct lfs_fsinfo *fsinfo) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_fs_stat(%p, %p)", (void*)lfs, (void*)fsinfo); + + err = lfs_fs_rawstat(lfs, fsinfo); + + LFS_TRACE("lfs_fs_stat -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} + lfs_ssize_t lfs_fs_size(lfs_t *lfs) { int err = LFS_LOCK(lfs->cfg); if (err) { diff --git a/lfs.h b/lfs.h index d919aabc..bbe64efe 100644 --- a/lfs.h +++ b/lfs.h @@ -301,6 +301,33 @@ struct lfs_info { char name[LFS_NAME_MAX+1]; }; +// Filesystem info structure +// +// Some of these can also be found in lfs_config, but the values here respect +// what was stored in the superblock during lfs_format. +struct lfs_fsinfo { + // Size of a logical block in bytes. + lfs_size_t block_size; + + // Number of logical blocks on the block device. + lfs_size_t block_count; + + // Number of blocks in use, this is the same as lfs_fs_size. + // + // Note: Result is best effort. If files share COW structures, the returned + // size may be larger than the filesystem actually is. + lfs_size_t block_usage; + + // Upper limit on the length of file names in bytes. + lfs_size_t name_max; + + // Upper limit on the size of files in bytes. + lfs_size_t file_max; + + // Upper limit on the size of custom attributes in bytes. + lfs_size_t attr_max; +}; + // Custom attribute structure, used to describe custom attributes // committed atomically during file writes. struct lfs_attr { @@ -683,6 +710,11 @@ int lfs_dir_rewind(lfs_t *lfs, lfs_dir_t *dir); /// Filesystem-level filesystem operations +// Find info about the filesystem +// +// Fills out the fsinfo structure. Returns a negative error code on failure. +int lfs_fs_stat(lfs_t *lfs, struct lfs_fsinfo *fsinfo); + // Finds the current size of the filesystem // // Note: Result is best effort. If files share COW structures, the returned diff --git a/tests/test_superblocks.toml b/tests/test_superblocks.toml index 2625ecc3..1e34b3f2 100644 --- a/tests/test_superblocks.toml +++ b/tests/test_superblocks.toml @@ -157,7 +157,13 @@ code = ''' lfs_format(&lfs, cfg) => 0; cfg->block_size = 0; + lfs_mount(&lfs, cfg) => 0; + struct lfs_fsinfo fsinfo; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_file_t file; lfs_file_open(&lfs, &file, "test", LFS_O_CREAT | LFS_O_EXCL | LFS_O_WRONLY) => 0; @@ -166,6 +172,10 @@ code = ''' lfs_unmount(&lfs) => 0; lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_file_open(&lfs, &file, "test", LFS_O_RDONLY) => 0; uint8_t buffer[256]; lfs_file_read(&lfs, &file, buffer, sizeof(buffer)) => 6; @@ -181,7 +191,48 @@ code = ''' lfs_format(&lfs, cfg) => 0; cfg->block_count = 0; + + lfs_mount(&lfs, cfg) => 0; + struct lfs_fsinfo fsinfo; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + + lfs_file_t file; + lfs_file_open(&lfs, &file, "test", + LFS_O_CREAT | LFS_O_EXCL | LFS_O_WRONLY) => 0; + lfs_file_write(&lfs, &file, "hello!", 6) => 6; + lfs_file_close(&lfs, &file) => 0; + lfs_unmount(&lfs) => 0; + + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + + lfs_file_open(&lfs, &file, "test", LFS_O_RDONLY) => 0; + uint8_t buffer[256]; + lfs_file_read(&lfs, &file, buffer, sizeof(buffer)) => 6; + lfs_file_close(&lfs, &file) => 0; + assert(memcmp(buffer, "hello!", 6) == 0); + lfs_unmount(&lfs) => 0; +''' + +# mount with unknown block_size+block_count +[cases.test_superblocks_unknown_block_size_count] +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + + cfg->block_size = 0; + cfg->block_count = 0; + lfs_mount(&lfs, cfg) => 0; + struct lfs_fsinfo fsinfo; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_file_t file; lfs_file_open(&lfs, &file, "test", LFS_O_CREAT | LFS_O_EXCL | LFS_O_WRONLY) => 0; @@ -190,6 +241,10 @@ code = ''' lfs_unmount(&lfs) => 0; lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_file_open(&lfs, &file, "test", LFS_O_RDONLY) => 0; uint8_t buffer[256]; lfs_file_read(&lfs, &file, buffer, sizeof(buffer)) => 6; From fc10790ebaf1fd7459c24cd9a6ed8d2a9d90d4b0 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 12 Nov 2022 14:31:52 -0600 Subject: [PATCH 6/9] Added more block_size/count inference tests and fixed a couple bugs - Need to drop cache during block_size search. - Removed "Corrupted" messages when root is not found to avoid too much debug output. This also gets rid of a long-term point of confusion arround "Corrupted" messages on a mount that is intended to fail. --- lfs.c | 10 +- tests/test_superblocks.toml | 331 ++++++++++++++++++++++++++++++++++-- 2 files changed, 327 insertions(+), 14 deletions(-) diff --git a/lfs.c b/lfs.c index 38d09273..8dfbedcc 100644 --- a/lfs.c +++ b/lfs.c @@ -1299,8 +1299,10 @@ static lfs_stag_t lfs_dir_fetchmatch(lfs_t *lfs, dir->rev = revs[(r+1)%2]; } - LFS_ERROR("Corrupted dir pair at {0x%"PRIx32", 0x%"PRIx32"}", - dir->pair[0], dir->pair[1]); + if (!lfs_pair_isnull(lfs->root)) { + LFS_ERROR("Corrupted dir pair at {0x%"PRIx32", 0x%"PRIx32"}", + dir->pair[0], dir->pair[1]); + } return LFS_ERR_CORRUPT; } @@ -4231,6 +4233,10 @@ static int lfs_rawmount(lfs_t *lfs, const struct lfs_config *cfg) { lfs->block_count /= lfs->block_size/lfs->erase_size; } + // make sure cached data from a different block_size doesn't cause + // problems, we should never visit the same mdir twice here anyways + lfs_cache_drop(lfs, &lfs->rcache); + // scan directory blocks for superblock and any global updates lfs_mdir_t dir = {.tail = {0, 1}}; lfs_block_t cycle = 0; diff --git a/tests/test_superblocks.toml b/tests/test_superblocks.toml index 1e34b3f2..67d13f6f 100644 --- a/tests/test_superblocks.toml +++ b/tests/test_superblocks.toml @@ -150,20 +150,54 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -# mount with unknown block_size -[cases.test_superblocks_unknown_block_size] +# mount with unknown block_size/block_count +[cases.test_superblocks_unknowns] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; + // known block_size/block_count + cfg->block_size = BLOCK_SIZE; + cfg->block_count = BLOCK_COUNT; + lfs_mount(&lfs, cfg) => 0; + struct lfs_fsinfo fsinfo; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + + // unknown block_count + cfg->block_size = BLOCK_SIZE; + cfg->block_count = 0; + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + + // unknown block_size cfg->block_size = 0; + cfg->block_count = ERASE_COUNT; + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + // unknown block_count/block_size + cfg->block_size = 0; + cfg->block_count = 0; lfs_mount(&lfs, cfg) => 0; - struct lfs_fsinfo fsinfo; lfs_fs_stat(&lfs, &fsinfo) => 0; assert(fsinfo.block_size == BLOCK_SIZE); assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + // do some work + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); lfs_file_t file; lfs_file_open(&lfs, &file, "test", LFS_O_CREAT | LFS_O_EXCL | LFS_O_WRONLY) => 0; @@ -175,7 +209,6 @@ code = ''' lfs_fs_stat(&lfs, &fsinfo) => 0; assert(fsinfo.block_size == BLOCK_SIZE); assert(fsinfo.block_count == BLOCK_COUNT); - lfs_file_open(&lfs, &file, "test", LFS_O_RDONLY) => 0; uint8_t buffer[256]; lfs_file_read(&lfs, &file, buffer, sizeof(buffer)) => 6; @@ -184,20 +217,63 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -# mount with unknown block_count -[cases.test_superblocks_unknown_block_count] +# mount with blocks larger than the erase_size +[cases.test_superblocks_larger_blocks] +defines.BLOCK_SIZE = [ + '2*ERASE_SIZE', + '4*ERASE_SIZE', + '(ERASE_COUNT/2)*ERASE_SIZE'] code = ''' lfs_t lfs; lfs_format(&lfs, cfg) => 0; + // known block_size/block_count + cfg->block_size = BLOCK_SIZE; + cfg->block_count = BLOCK_COUNT; + lfs_mount(&lfs, cfg) => 0; + struct lfs_fsinfo fsinfo; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + + // incorrect block_size + cfg->block_size = ERASE_SIZE; cfg->block_count = 0; + lfs_mount(&lfs, cfg) => LFS_ERR_INVAL; + // unknown block_count + cfg->block_size = BLOCK_SIZE; + cfg->block_count = 0; + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + + // unknown block_size + cfg->block_size = 0; + cfg->block_count = ERASE_COUNT; + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + + // unknown block_count/block_size + cfg->block_size = 0; + cfg->block_count = 0; lfs_mount(&lfs, cfg) => 0; - struct lfs_fsinfo fsinfo; lfs_fs_stat(&lfs, &fsinfo) => 0; assert(fsinfo.block_size == BLOCK_SIZE); assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + // do some work + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); lfs_file_t file; lfs_file_open(&lfs, &file, "test", LFS_O_CREAT | LFS_O_EXCL | LFS_O_WRONLY) => 0; @@ -209,7 +285,6 @@ code = ''' lfs_fs_stat(&lfs, &fsinfo) => 0; assert(fsinfo.block_size == BLOCK_SIZE); assert(fsinfo.block_count == BLOCK_COUNT); - lfs_file_open(&lfs, &file, "test", LFS_O_RDONLY) => 0; uint8_t buffer[256]; lfs_file_read(&lfs, &file, buffer, sizeof(buffer)) => 6; @@ -218,21 +293,119 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -# mount with unknown block_size+block_count -[cases.test_superblocks_unknown_block_size_count] +# mount with blocks smaller than the erase_size +[cases.test_superblocks_smaller_blocks] +defines.FORMAT_BLOCK_SIZE = 'ERASE_SIZE/2' +defines.FORMAT_BLOCK_COUNT = 'BLOCK_COUNT * (BLOCK_SIZE/FORMAT_BLOCK_SIZE)' +in = 'lfs.c' code = ''' lfs_t lfs; - lfs_format(&lfs, cfg) => 0; + lfs_init(&lfs, cfg) => 0; + lfs.block_size = BLOCK_SIZE; + lfs.block_count = BLOCK_COUNT; + + lfs_mdir_t root = { + .pair = {0, 0}, // make sure this goes into block 0 + .rev = 0, + .off = sizeof(uint32_t), + .etag = 0xffffffff, + .count = 0, + .tail = {LFS_BLOCK_NULL, LFS_BLOCK_NULL}, + .erased = false, + .split = false, + }; + + lfs_superblock_t superblock = { + .version = LFS_DISK_VERSION, + .block_size = FORMAT_BLOCK_SIZE, + .block_count = FORMAT_BLOCK_COUNT, + .name_max = LFS_NAME_MAX, + .file_max = LFS_FILE_MAX, + .attr_max = LFS_ATTR_MAX, + }; + + lfs_superblock_tole32(&superblock); + lfs_dir_commit(&lfs, &root, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_CREATE, 0, 0), NULL}, + {LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), "littlefs"}, + {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), + &superblock})) => 0; + + // known block_size/block_count + cfg->block_size = BLOCK_SIZE; + cfg->block_count = BLOCK_COUNT; + lfs_mount(&lfs, cfg) => LFS_ERR_INVAL; + // unknown block_count + cfg->block_size = BLOCK_SIZE; + cfg->block_count = 0; + lfs_mount(&lfs, cfg) => LFS_ERR_INVAL; + + // unknown block_size + cfg->block_size = 0; + cfg->block_count = ERASE_COUNT; + lfs_mount(&lfs, cfg) => LFS_ERR_INVAL; + + // unknown block_count/block_size cfg->block_size = 0; cfg->block_count = 0; + lfs_mount(&lfs, cfg) => LFS_ERR_INVAL; +''' +# mount with blocks fewer than the erase_count +[cases.test_superblocks_fewer_blocks] +defines.BLOCK_COUNT = ['ERASE_COUNT/2', 'ERASE_COUNT/4', '2'] +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + + // known block_size/block_count + cfg->block_size = BLOCK_SIZE; + cfg->block_count = BLOCK_COUNT; lfs_mount(&lfs, cfg) => 0; struct lfs_fsinfo fsinfo; lfs_fs_stat(&lfs, &fsinfo) => 0; assert(fsinfo.block_size == BLOCK_SIZE); assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + + // incorrect block_count + cfg->block_size = BLOCK_SIZE; + cfg->block_count = ERASE_COUNT; + lfs_mount(&lfs, cfg) => LFS_ERR_INVAL; + + // unknown block_count + cfg->block_size = BLOCK_SIZE; + cfg->block_count = 0; + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + + // unknown block_size + cfg->block_size = 0; + cfg->block_count = BLOCK_COUNT; + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + // unknown block_count/block_size + cfg->block_size = 0; + cfg->block_count = 0; + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + + // do some work + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); lfs_file_t file; lfs_file_open(&lfs, &file, "test", LFS_O_CREAT | LFS_O_EXCL | LFS_O_WRONLY) => 0; @@ -244,7 +417,6 @@ code = ''' lfs_fs_stat(&lfs, &fsinfo) => 0; assert(fsinfo.block_size == BLOCK_SIZE); assert(fsinfo.block_count == BLOCK_COUNT); - lfs_file_open(&lfs, &file, "test", LFS_O_RDONLY) => 0; uint8_t buffer[256]; lfs_file_read(&lfs, &file, buffer, sizeof(buffer)) => 6; @@ -252,3 +424,138 @@ code = ''' assert(memcmp(buffer, "hello!", 6) == 0); lfs_unmount(&lfs) => 0; ''' + +# mount with more blocks than the erase_count +[cases.test_superblocks_more_blocks] +defines.FORMAT_BLOCK_COUNT = '2*ERASE_COUNT' +in = 'lfs.c' +code = ''' + lfs_t lfs; + lfs_init(&lfs, cfg) => 0; + lfs.block_size = BLOCK_SIZE; + lfs.block_count = BLOCK_COUNT; + + lfs_mdir_t root = { + .pair = {0, 0}, // make sure this goes into block 0 + .rev = 0, + .off = sizeof(uint32_t), + .etag = 0xffffffff, + .count = 0, + .tail = {LFS_BLOCK_NULL, LFS_BLOCK_NULL}, + .erased = false, + .split = false, + }; + + lfs_superblock_t superblock = { + .version = LFS_DISK_VERSION, + .block_size = BLOCK_SIZE, + .block_count = FORMAT_BLOCK_COUNT, + .name_max = LFS_NAME_MAX, + .file_max = LFS_FILE_MAX, + .attr_max = LFS_ATTR_MAX, + }; + + lfs_superblock_tole32(&superblock); + lfs_dir_commit(&lfs, &root, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_CREATE, 0, 0), NULL}, + {LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), "littlefs"}, + {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), + &superblock})) => 0; + + // known block_size/block_count + cfg->block_size = BLOCK_SIZE; + cfg->block_count = BLOCK_COUNT; + lfs_mount(&lfs, cfg) => LFS_ERR_INVAL; + + // unknown block_size + cfg->block_size = 0; + cfg->block_count = ERASE_COUNT; + lfs_mount(&lfs, cfg) => LFS_ERR_INVAL; +''' + +# the real challenge is finding superblock 1 when block_size is unknown, +# test for this explicitly +[cases.test_superblocks_unknown_superblock1] +in = 'lfs.c' +defines.BLOCK_SIZE = [ + 'ERASE_SIZE', + '2*ERASE_SIZE', + '4*ERASE_SIZE', + '(ERASE_COUNT/2)*ERASE_SIZE'] +code = ''' + lfs_t lfs; + lfs_init(&lfs, cfg) => 0; + lfs.block_size = BLOCK_SIZE; + lfs.block_count = BLOCK_COUNT; + + uint8_t buffer[lfs_max(READ_SIZE, sizeof(uint32_t))]; + memset(buffer, 0, lfs_max(READ_SIZE, sizeof(uint32_t))); + cfg->read(cfg, 0, 0, buffer, lfs_max(READ_SIZE, sizeof(uint32_t))) => 0; + uint32_t rev; + memcpy(&rev, buffer, sizeof(uint32_t)); + rev = lfs_fromle32(rev); + + lfs_mdir_t root = { + .pair = {1, 1}, // make sure this goes into block 1 + .rev = rev, + .off = sizeof(uint32_t), + .etag = 0xffffffff, + .count = 0, + .tail = {LFS_BLOCK_NULL, LFS_BLOCK_NULL}, + .erased = false, + .split = false, + }; + + lfs_superblock_t superblock = { + .version = LFS_DISK_VERSION, + .block_size = BLOCK_SIZE, + .block_count = BLOCK_COUNT, + .name_max = LFS_NAME_MAX, + .file_max = LFS_FILE_MAX, + .attr_max = LFS_ATTR_MAX, + }; + + lfs_superblock_tole32(&superblock); + lfs_dir_commit(&lfs, &root, LFS_MKATTRS( + {LFS_MKTAG(LFS_TYPE_CREATE, 0, 0), NULL}, + {LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), "littlefs"}, + {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), + &superblock})) => 0; + + // known block_size/block_count + cfg->block_size = BLOCK_SIZE; + cfg->block_count = BLOCK_COUNT; + lfs_mount(&lfs, cfg) => 0; + struct lfs_fsinfo fsinfo; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + + // unknown block_count + cfg->block_size = BLOCK_SIZE; + cfg->block_count = 0; + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + + // unknown block_size + cfg->block_size = 0; + cfg->block_count = ERASE_COUNT; + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + + // unknown block_count/block_size + cfg->block_size = 0; + cfg->block_count = 0; + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; +''' From 34b5b395514dbb245ae3dda3f0bb67564afbebcb Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 13 Nov 2022 13:09:21 -0600 Subject: [PATCH 7/9] Added lfs_fs_grow for growing the filesystem to a different block_count The initial implementation for this was provided by kaetemi, originally as a mount flag. However, it has been modified here to be self-contained in an explicit runtime function that can be called after mount. The reasons for an explicit function: 1. lfs_mount stays a strictly readonly operation, and avoids pulling in all of the write machinery. 2. filesystem-wide operations such as lfs_fs_grow can be a bit risky, and irreversable. The action of growing the filesystem should be very intentional. --- One concern with this change is that this will be the first function that changes metadata in the superblock. This might break tools that expect the first valid superblock entry to contain the most recent metadata. --- lfs.c | 53 ++++++++++++++++++ lfs.h | 10 ++++ tests/test_superblocks.toml | 105 ++++++++++++++++++++++++++++++++++++ 3 files changed, 168 insertions(+) diff --git a/lfs.c b/lfs.c index 8dfbedcc..e46b9867 100644 --- a/lfs.c +++ b/lfs.c @@ -4846,6 +4846,43 @@ static int lfs_fs_rawstat(lfs_t *lfs, struct lfs_fsinfo *fsinfo) { return 0; } +int lfs_fs_rawgrow(lfs_t *lfs, lfs_size_t block_count) { + // shrinking is not supported + LFS_ASSERT(block_count >= lfs->block_count); + + if (block_count > lfs->block_count) { + lfs->block_count = block_count; + + // fetch the root + lfs_mdir_t root; + int err = lfs_dir_fetch(lfs, &root, lfs->root); + if (err) { + return err; + } + + // update the superblock + lfs_superblock_t superblock; + lfs_stag_t tag = lfs_dir_get(lfs, &root, LFS_MKTAG(0x7ff, 0x3ff, 0), + LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), + &superblock); + if (tag < 0) { + return tag; + } + lfs_superblock_fromle32(&superblock); + + superblock.block_count = lfs->block_count; + + lfs_superblock_tole32(&superblock); + err = lfs_dir_commit(lfs, &root, LFS_MKATTRS( + {tag, &superblock})); + if (err) { + return err; + } + } + + return 0; +} + #ifdef LFS_MIGRATE ////// Migration from littelfs v1 below this ////// @@ -6012,6 +6049,22 @@ int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void *, lfs_block_t), void *data) { return err; } +#ifndef LFS_READONLY +int lfs_fs_grow(lfs_t *lfs, lfs_size_t block_count) { + int err = LFS_LOCK(lfs->cfg); + if (err) { + return err; + } + LFS_TRACE("lfs_fs_grow(%p, %"PRIu32")", (void*)lfs, block_count); + + err = lfs_fs_rawgrow(lfs, block_count); + + LFS_TRACE("lfs_fs_grow -> %d", err); + LFS_UNLOCK(lfs->cfg); + return err; +} +#endif + #ifdef LFS_MIGRATE int lfs_migrate(lfs_t *lfs, const struct lfs_config *cfg) { int err = LFS_LOCK(cfg); diff --git a/lfs.h b/lfs.h index bbe64efe..d9bfe8c0 100644 --- a/lfs.h +++ b/lfs.h @@ -732,6 +732,16 @@ lfs_ssize_t lfs_fs_size(lfs_t *lfs); // Returns a negative error code on failure. int lfs_fs_traverse(lfs_t *lfs, int (*cb)(void*, lfs_block_t), void *data); +#ifndef LFS_READONLY +// Grows the filesystem to a new size, updating the superblock with the new +// block count. +// +// Note: This is irreversible. +// +// Returns a negative error code on failure. +int lfs_fs_grow(lfs_t *lfs, lfs_size_t block_count); +#endif + #ifndef LFS_READONLY #ifdef LFS_MIGRATE // Attempts to migrate a previous version of littlefs diff --git a/tests/test_superblocks.toml b/tests/test_superblocks.toml index 67d13f6f..6ff25345 100644 --- a/tests/test_superblocks.toml +++ b/tests/test_superblocks.toml @@ -559,3 +559,108 @@ code = ''' assert(fsinfo.block_count == BLOCK_COUNT); lfs_unmount(&lfs) => 0; ''' + +# mount and grow the filesystem +[cases.test_superblocks_grow] +defines.BLOCK_COUNT = ['ERASE_COUNT/2', 'ERASE_COUNT/4', '2'] +defines.BLOCK_COUNT_2 = 'ERASE_COUNT' +defines.KNOWN_BLOCK_COUNT = [true, false] +code = ''' + lfs_t lfs; + lfs_format(&lfs, cfg) => 0; + + if (KNOWN_BLOCK_COUNT) { + cfg->block_count = BLOCK_COUNT; + } else { + cfg->block_count = 0; + } + + // mount with block_size < erase_size + lfs_mount(&lfs, cfg) => 0; + struct lfs_fsinfo fsinfo; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + + // same size is a noop + lfs_mount(&lfs, cfg) => 0; + lfs_fs_grow(&lfs, BLOCK_COUNT) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT); + lfs_unmount(&lfs) => 0; + + // grow to new size + lfs_mount(&lfs, cfg) => 0; + lfs_fs_grow(&lfs, BLOCK_COUNT_2) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT_2); + lfs_unmount(&lfs) => 0; + + if (KNOWN_BLOCK_COUNT) { + cfg->block_count = BLOCK_COUNT_2; + } else { + cfg->block_count = 0; + } + + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT_2); + lfs_unmount(&lfs) => 0; + + // mounting with the previous size should fail + cfg->block_count = BLOCK_COUNT; + lfs_mount(&lfs, cfg) => LFS_ERR_INVAL; + + if (KNOWN_BLOCK_COUNT) { + cfg->block_count = BLOCK_COUNT_2; + } else { + cfg->block_count = 0; + } + + // same size is a noop + lfs_mount(&lfs, cfg) => 0; + lfs_fs_grow(&lfs, BLOCK_COUNT_2) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT_2); + lfs_unmount(&lfs) => 0; + + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT_2); + lfs_unmount(&lfs) => 0; + + // do some work + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT_2); + lfs_file_t file; + lfs_file_open(&lfs, &file, "test", + LFS_O_CREAT | LFS_O_EXCL | LFS_O_WRONLY) => 0; + lfs_file_write(&lfs, &file, "hello!", 6) => 6; + lfs_file_close(&lfs, &file) => 0; + lfs_unmount(&lfs) => 0; + + lfs_mount(&lfs, cfg) => 0; + lfs_fs_stat(&lfs, &fsinfo) => 0; + assert(fsinfo.block_size == BLOCK_SIZE); + assert(fsinfo.block_count == BLOCK_COUNT_2); + lfs_file_open(&lfs, &file, "test", LFS_O_RDONLY) => 0; + uint8_t buffer[256]; + lfs_file_read(&lfs, &file, buffer, sizeof(buffer)) => 6; + lfs_file_close(&lfs, &file) => 0; + assert(memcmp(buffer, "hello!", 6) == 0); + lfs_unmount(&lfs) => 0; +''' From f198c855f6f5289469b28f3ca437e1f861b85317 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sun, 13 Nov 2022 21:30:57 -0600 Subject: [PATCH 8/9] Added benchmarks over mounting an unformatted disk These locally show the expected O(d(n)) growth of the block_size search when block_count is known. A rough estimate of runtime for a 1 GiB device with a 90 MiB/s read bandwidth (using the mt25q for rough numbers): known bs/bc => ~1.44 us known block_size => ~1.44 us known block_count => ~125 us unknown bs/bc => ~376 ms --- bd/lfs_emubd.c | 4 ++- bd/lfs_filebd.c | 4 ++- bd/lfs_rambd.c | 4 ++- benches/bench_superblock.toml | 50 +++++++++++++++++++++++++++-------- 4 files changed, 48 insertions(+), 14 deletions(-) diff --git a/bd/lfs_emubd.c b/bd/lfs_emubd.c index 5ab8b082..de3075ec 100644 --- a/bd/lfs_emubd.c +++ b/bd/lfs_emubd.c @@ -220,7 +220,9 @@ int lfs_emubd_read(const struct lfs_config *cfg, lfs_block_t block, lfs_emubd_t *bd = cfg->context; // check if read is valid - LFS_ASSERT(block < bd->cfg->erase_count); + if (block >= bd->cfg->erase_count) { + return LFS_ERR_INVAL; + } LFS_ASSERT(off % bd->cfg->read_size == 0); LFS_ASSERT(size % bd->cfg->read_size == 0); LFS_ASSERT(off+size <= bd->cfg->erase_size); diff --git a/bd/lfs_filebd.c b/bd/lfs_filebd.c index 4ff25d44..5f874531 100644 --- a/bd/lfs_filebd.c +++ b/bd/lfs_filebd.c @@ -70,7 +70,9 @@ int lfs_filebd_read(const struct lfs_config *cfg, lfs_block_t block, lfs_filebd_t *bd = cfg->context; // check if read is valid - LFS_ASSERT(block < bd->cfg->erase_count); + if (block >= bd->cfg->erase_count) { + return LFS_ERR_INVAL; + } LFS_ASSERT(off % bd->cfg->read_size == 0); LFS_ASSERT(size % bd->cfg->read_size == 0); LFS_ASSERT(off+size <= bd->cfg->erase_size); diff --git a/bd/lfs_rambd.c b/bd/lfs_rambd.c index a6a05727..1b29e336 100644 --- a/bd/lfs_rambd.c +++ b/bd/lfs_rambd.c @@ -60,7 +60,9 @@ int lfs_rambd_read(const struct lfs_config *cfg, lfs_block_t block, lfs_rambd_t *bd = cfg->context; // check if read is valid - LFS_ASSERT(block < bd->cfg->erase_count); + if (block >= bd->cfg->erase_count) { + return LFS_ERR_INVAL; + } LFS_ASSERT(off % bd->cfg->read_size == 0); LFS_ASSERT(size % bd->cfg->read_size == 0); LFS_ASSERT(off+size <= bd->cfg->erase_size); diff --git a/benches/bench_superblock.toml b/benches/bench_superblock.toml index 37659d47..ff013700 100644 --- a/benches/bench_superblock.toml +++ b/benches/bench_superblock.toml @@ -1,4 +1,15 @@ -[cases.bench_superblocks_found] +[cases.bench_superblocks_format] +code = ''' + lfs_t lfs; + + BENCH_START(); + lfs_format(&lfs, cfg) => 0; + BENCH_STOP(); +''' + +[cases.bench_superblocks_mount] +defines.KNOWN_BLOCK_SIZE = [true, false] +defines.KNOWN_BLOCK_COUNT = [true, false] # support benchmarking with files defines.N = [0, 1024] defines.FILE_SIZE = 8 @@ -28,6 +39,18 @@ code = ''' } lfs_unmount(&lfs) => 0; + if (KNOWN_BLOCK_SIZE) { + cfg->block_size = BLOCK_SIZE; + } else { + cfg->block_size = 0; + } + + if (KNOWN_BLOCK_COUNT) { + cfg->block_count = BLOCK_COUNT; + } else { + cfg->block_count = 0; + } + BENCH_START(); lfs_mount(&lfs, cfg) => 0; BENCH_STOP(); @@ -35,22 +58,27 @@ code = ''' lfs_unmount(&lfs) => 0; ''' -[cases.bench_superblocks_missing] +[cases.bench_superblocks_mount_missing] +defines.KNOWN_BLOCK_SIZE = [true, false] +defines.KNOWN_BLOCK_COUNT = [true, false] code = ''' lfs_t lfs; - BENCH_START(); - int err = lfs_mount(&lfs, cfg); - assert(err != 0); - BENCH_STOP(); -''' + if (KNOWN_BLOCK_SIZE) { + cfg->block_size = BLOCK_SIZE; + } else { + cfg->block_size = 0; + } -[cases.bench_superblocks_format] -code = ''' - lfs_t lfs; + if (KNOWN_BLOCK_COUNT) { + cfg->block_count = BLOCK_COUNT; + } else { + cfg->block_count = 0; + } BENCH_START(); - lfs_format(&lfs, cfg) => 0; + int err = lfs_mount(&lfs, cfg); + assert(err != 0); BENCH_STOP(); ''' From 689bdc8e06b65fba17c0f2e8ce8b93e7e9160029 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Tue, 6 Dec 2022 00:43:10 -0600 Subject: [PATCH 9/9] Fixed handful of block_size-related bugs found by CI - Valgrind-detected memory leak in test_superblocks. - Valgrind-detected uninitialized lfs->block_size in lfs_init caused by an unintended sed replacement. - Said sed replacement also changed the implementation of the v1 portion of lfs_migrate, this is out of scope and caused migrate to fail, which fortunately CI noticed. - We also need to copy the block_size/block_count configs in v1 so that low-level bd operations still work. - le-conversion in test_evil now matters due to difference in LFS_ERR_CORRUPT/LFS_ERR_INVAL paths during lfs_mount. This is good because we're actually testing for loop detection not pointer out-of-bounds as intended. --- lfs.c | 19 ++++++++++++++----- tests/test_evil.toml | 5 +++-- tests/test_superblocks.toml | 3 +++ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/lfs.c b/lfs.c index e46b9867..8ed8b2d9 100644 --- a/lfs.c +++ b/lfs.c @@ -4087,7 +4087,7 @@ static int lfs_init(lfs_t *lfs, const struct lfs_config *cfg) { lfs->attr_max = LFS_ATTR_MAX; } - LFS_ASSERT(lfs->cfg->metadata_max <= lfs->block_size); + LFS_ASSERT(lfs->cfg->metadata_max <= lfs->cfg->block_size); // setup default state lfs->root[0] = LFS_BLOCK_NULL; @@ -4139,8 +4139,8 @@ static int lfs_rawformat(lfs_t *lfs, const struct lfs_config *cfg) { if (!lfs->block_size) { lfs->block_size = lfs->erase_size; } + LFS_ASSERT(lfs->cfg->block_count != 0); lfs->block_count = lfs->cfg->block_count; - LFS_ASSERT(lfs->block_count != 0); // check that the block size is large enough to fit ctz pointers LFS_ASSERT(4*lfs_npw2(0xffffffff / (lfs->block_size-2*4)) @@ -4846,6 +4846,7 @@ static int lfs_fs_rawstat(lfs_t *lfs, struct lfs_fsinfo *fsinfo) { return 0; } +#ifndef LFS_READONLY int lfs_fs_rawgrow(lfs_t *lfs, lfs_size_t block_count) { // shrinking is not supported LFS_ASSERT(block_count >= lfs->block_count); @@ -4882,6 +4883,7 @@ int lfs_fs_rawgrow(lfs_t *lfs, lfs_size_t block_count) { return 0; } +#endif #ifdef LFS_MIGRATE ////// Migration from littelfs v1 below this ////// @@ -5057,7 +5059,7 @@ static int lfs1_dir_fetch(lfs_t *lfs, } if ((0x7fffffff & test.size) < sizeof(test)+4 || - (0x7fffffff & test.size) > lfs->block_size) { + (0x7fffffff & test.size) > lfs->cfg->block_size) { continue; } @@ -5245,6 +5247,13 @@ static int lfs1_mount(lfs_t *lfs, struct lfs1 *lfs1, return err; } + // setup block_size, v1 did not support block_size searching, so + // block_size and block_count are required + LFS_ASSERT(lfs->cfg->block_size != 0); + LFS_ASSERT(lfs->cfg->block_count != 0); + lfs->block_size = lfs->cfg->block_size; + lfs->block_count = lfs->cfg->block_count; + lfs->lfs1 = lfs1; lfs->lfs1->root[0] = LFS_BLOCK_NULL; lfs->lfs1->root[1] = LFS_BLOCK_NULL; @@ -5491,8 +5500,8 @@ static int lfs_rawmigrate(lfs_t *lfs, const struct lfs_config *cfg) { lfs_superblock_t superblock = { .version = LFS_DISK_VERSION, - .block_size = lfs->block_size, - .block_count = lfs->block_count, + .block_size = lfs->cfg->block_size, + .block_count = lfs->cfg->block_count, .name_max = lfs->name_max, .file_max = lfs->file_max, .attr_max = lfs->attr_max, diff --git a/tests/test_evil.toml b/tests/test_evil.toml index 1dde8650..a15d670a 100644 --- a/tests/test_evil.toml +++ b/tests/test_evil.toml @@ -237,7 +237,7 @@ code = ''' lfs_dir_fetch(&lfs, &mdir, (lfs_block_t[2]){0, 1}) => 0; lfs_dir_commit(&lfs, &mdir, LFS_MKATTRS( {LFS_MKTAG(LFS_TYPE_HARDTAIL, 0x3ff, 8), - (lfs_block_t[2]){0, 1}})) => 0; + (lfs_block_t[2]){lfs_tole32(0), lfs_tole32(1)}})) => 0; lfs_deinit(&lfs) => 0; // test that mount fails gracefully @@ -268,7 +268,7 @@ code = ''' lfs_dir_fetch(&lfs, &mdir, pair) => 0; lfs_dir_commit(&lfs, &mdir, LFS_MKATTRS( {LFS_MKTAG(LFS_TYPE_HARDTAIL, 0x3ff, 8), - (lfs_block_t[2]){0, 1}})) => 0; + (lfs_block_t[2]){lfs_tole32(0), lfs_tole32(1)}})) => 0; lfs_deinit(&lfs) => 0; // test that mount fails gracefully @@ -297,6 +297,7 @@ code = ''' lfs_pair_fromle32(pair); // change tail-pointer to point to ourself lfs_dir_fetch(&lfs, &mdir, pair) => 0; + lfs_pair_tole32(pair); lfs_dir_commit(&lfs, &mdir, LFS_MKATTRS( {LFS_MKTAG(LFS_TYPE_HARDTAIL, 0x3ff, 8), pair})) => 0; lfs_deinit(&lfs) => 0; diff --git a/tests/test_superblocks.toml b/tests/test_superblocks.toml index 6ff25345..881dd098 100644 --- a/tests/test_superblocks.toml +++ b/tests/test_superblocks.toml @@ -330,6 +330,7 @@ code = ''' {LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), "littlefs"}, {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), &superblock})) => 0; + lfs_deinit(&lfs) => 0; // known block_size/block_count cfg->block_size = BLOCK_SIZE; @@ -461,6 +462,7 @@ code = ''' {LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), "littlefs"}, {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), &superblock})) => 0; + lfs_deinit(&lfs) => 0; // known block_size/block_count cfg->block_size = BLOCK_SIZE; @@ -521,6 +523,7 @@ code = ''' {LFS_MKTAG(LFS_TYPE_SUPERBLOCK, 0, 8), "littlefs"}, {LFS_MKTAG(LFS_TYPE_INLINESTRUCT, 0, sizeof(superblock)), &superblock})) => 0; + lfs_deinit(&lfs) => 0; // known block_size/block_count cfg->block_size = BLOCK_SIZE;