From 836e23895a0e9205e6b530112dcc9374306417d7 Mon Sep 17 00:00:00 2001 From: Christopher Haster Date: Sat, 17 Mar 2018 10:28:14 -0500 Subject: [PATCH] Shoehorned in hacky implementation of inline files Proof-of-concept implementation of inline files that stores the file's content directly in its parent's directory pair. Inline files are indicated by a different type stored in an entry's struct field, and take advantage of resizable entries. Where a normal file's entry would normally hold the reference to the CTZ skip-list, an inline file's entry contains the contents of the actual file. Unfortunately, storing the inline file on disk is the easy part. We also need to manage inline files in the internals of littlefs and provide the same operations that we do on normal files, all while reusing as much code as possible to avoid a significant increase in code cost. There is a relatively simple, though maybe a bit hacky, solution here. If a file fits entirely in a cache line, the file logic never actually has to go to disk. This means we can just give the file a "pretend" block (hopefully one that would assert if ever written to), and carry out file operations as normal, as long as we catch the file before it exceeds the cache line and write out the file to an actual disk. --- lfs.c | 163 ++++++++++++++++++++++++++++++++------------- lfs.h | 28 +++++--- tests/test_dirs.sh | 2 +- 3 files changed, 133 insertions(+), 60 deletions(-) diff --git a/lfs.c b/lfs.c index cfbc2b91f82..d0128636cf1 100644 --- a/lfs.c +++ b/lfs.c @@ -24,7 +24,7 @@ static int lfs_cache_read(lfs_t *lfs, lfs_cache_t *rcache, const lfs_cache_t *pcache, lfs_block_t block, lfs_off_t off, void *buffer, lfs_size_t size) { uint8_t *data = buffer; - LFS_ASSERT(block < lfs->cfg->block_count); + LFS_ASSERT(block != 0xffffffff); while (size > 0) { if (pcache && block == pcache->block && off >= pcache->off && @@ -68,6 +68,7 @@ static int lfs_cache_read(lfs_t *lfs, lfs_cache_t *rcache, } // load to cache, first condition can no longer fail + LFS_ASSERT(block < lfs->cfg->block_count); rcache->block = block; rcache->off = off - (off % lfs->cfg->read_size); int err = lfs->cfg->read(lfs->cfg, rcache->block, @@ -121,6 +122,7 @@ static int lfs_cache_crc(lfs_t *lfs, lfs_cache_t *rcache, static int lfs_cache_flush(lfs_t *lfs, lfs_cache_t *pcache, lfs_cache_t *rcache) { if (pcache->block != 0xffffffff) { + LFS_ASSERT(pcache->block < lfs->cfg->block_count); int err = lfs->cfg->prog(lfs->cfg, pcache->block, pcache->off, pcache->buffer, lfs->cfg->prog_size); if (err) { @@ -149,7 +151,7 @@ static int lfs_cache_prog(lfs_t *lfs, lfs_cache_t *pcache, lfs_cache_t *rcache, lfs_block_t block, lfs_off_t off, const void *buffer, lfs_size_t size) { const uint8_t *data = buffer; - LFS_ASSERT(block < lfs->cfg->block_count); + LFS_ASSERT(block != 0xffffffff); while (size > 0) { if (block == pcache->block && off >= pcache->off && @@ -181,6 +183,7 @@ static int lfs_cache_prog(lfs_t *lfs, lfs_cache_t *pcache, if (off % lfs->cfg->prog_size == 0 && size >= lfs->cfg->prog_size) { // bypass pcache? + LFS_ASSERT(block < lfs->cfg->block_count); lfs_size_t diff = size - (size % lfs->cfg->prog_size); int err = lfs->cfg->prog(lfs->cfg, block, off, data, diff); if (err) { @@ -240,6 +243,7 @@ static int lfs_bd_crc(lfs_t *lfs, lfs_block_t block, } static int lfs_bd_erase(lfs_t *lfs, lfs_block_t block) { + LFS_ASSERT(block < lfs->cfg->block_count); return lfs->cfg->erase(lfs->cfg, block); } @@ -851,7 +855,7 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir, } static int lfs_dir_next(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) { - while (dir->off + sizeof(entry->d) > (0x7fffffff & dir->d.size)-4) { + while (dir->off >= (0x7fffffff & dir->d.size)-4) { if (!(0x80000000 & dir->d.size)) { entry->off = dir->off; return LFS_ERR_NOENT; @@ -942,8 +946,8 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir, return err; } - if (((0x7f & entry->d.type) != (LFS_STRUCT_CTZ | LFS_TYPE_REG) && - (0x7f & entry->d.type) != (LFS_STRUCT_DIR | LFS_TYPE_DIR)) || + if (((0xf & entry->d.type) != LFS_TYPE_REG && + (0xf & entry->d.type) != LFS_TYPE_DIR) || entry->d.nlen != pathlen) { continue; } @@ -1126,8 +1130,8 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { return (err == LFS_ERR_NOENT) ? 0 : err; } - if ((0x7f & entry.d.type) != (LFS_STRUCT_CTZ | LFS_TYPE_REG) && - (0x7f & entry.d.type) != (LFS_STRUCT_DIR | LFS_TYPE_DIR)) { + if ((0xf & entry.d.type) != LFS_TYPE_REG && + (0xf & entry.d.type) != LFS_TYPE_DIR) { continue; } @@ -1149,8 +1153,10 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) { } info->type = 0xf & entry.d.type; - if (info->type == LFS_TYPE_REG) { + if (entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) { info->size = entry.d.u.file.size; + } else if (entry.d.type == (LFS_STRUCT_INLINE | LFS_TYPE_REG)) { + info->size = entry.d.elen; } int err = lfs_bd_read(lfs, dir->pair[0], @@ -1424,18 +1430,16 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } // create entry to remember name - entry.d.type = LFS_STRUCT_CTZ | LFS_TYPE_REG; - entry.d.elen = sizeof(entry.d) - 4; + entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG; + entry.d.elen = 0; entry.d.alen = 0; entry.d.nlen = strlen(path); - entry.d.u.file.head = 0xffffffff; - entry.d.u.file.size = 0; entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; err = lfs_dir_append(lfs, &cwd, &entry, &(struct lfs_region){ - 0, +sizeof(entry.d), - lfs_commit_mem, &entry.d, sizeof(entry.d), + 0, +4, + lfs_commit_mem, &entry.d, 4, &(struct lfs_region){ 0, +entry.d.nlen, lfs_commit_mem, path, entry.d.nlen}}); @@ -1481,6 +1485,22 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file, } } + // load inline files + if ((0x70 & entry.d.type) == LFS_STRUCT_INLINE) { + file->head = 0xfffffffe; + file->size = entry.d.elen; + file->flags |= LFS_F_INLINE; + file->cache.block = file->head; + file->cache.off = 0; + err = lfs_bd_read(lfs, cwd.pair[0], + entry.off + 4, + file->cache.buffer, file->size); + if (err) { + lfs_free(file->cache.buffer); + return err; + } + } + // add to list of files file->next = lfs->files; lfs->files = file; @@ -1556,6 +1576,20 @@ static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) { } static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) { + if (file->flags & LFS_F_INLINE) { + // do nothing since we won't need the cache for anything else + if (file->flags & LFS_F_READING) { + file->flags &= ~LFS_F_READING; + } + + if (file->flags & LFS_F_WRITING) { + file->flags &= ~LFS_F_WRITING; + file->flags |= LFS_F_DIRTY; + } + + return 0; + } + if (file->flags & LFS_F_READING) { // just drop read cache file->cache.block = 0xffffffff; @@ -1642,6 +1676,7 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { return err; } + // TODO entry read? lfs_entry_t entry = {.off = file->poff}; err = lfs_bd_read(lfs, cwd.pair[0], entry.off, &entry.d, sizeof(entry.d)); @@ -1649,19 +1684,35 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) { if (err) { return err; } + LFS_ASSERT((0xf & entry.d.type) == LFS_TYPE_REG); + entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen; - LFS_ASSERT(entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)); - entry.d.u.file.head = file->head; - entry.d.u.file.size = file->size; + if (file->flags & LFS_F_INLINE) { + file->size = lfs_max(file->pos, file->size); + lfs_ssize_t diff = file->size - entry.d.elen; + entry.d.elen = file->size; - lfs_entry_tole32(&entry.d); - err = lfs_dir_update(lfs, &cwd, &entry, - &(struct lfs_region){ - 0, 0, - lfs_commit_mem, &entry.d, sizeof(entry.d)}); - lfs_entry_fromle32(&entry.d); - if (err) { - return err; + err = lfs_dir_update(lfs, &cwd, &entry, + &(struct lfs_region){ + 0, 0, + lfs_commit_mem, &entry.d, 4, + &(struct lfs_region){ + 4, diff, + lfs_commit_mem, file->cache.buffer, file->size}}); + if (err) { + return err; + } + } else { + entry.d.u.file.head = file->head; + entry.d.u.file.size = file->size; + + err = lfs_dir_update(lfs, &cwd, &entry, + &(struct lfs_region){ + 0, 0, + lfs_commit_mem, &entry.d, sizeof(entry.d)}); + if (err) { + return err; + } } file->flags &= ~LFS_F_DIRTY; @@ -1699,11 +1750,16 @@ lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file, // check if we need a new block if (!(file->flags & LFS_F_READING) || file->off == lfs->cfg->block_size) { - int err = lfs_ctz_find(lfs, &file->cache, NULL, - file->head, file->size, - file->pos, &file->block, &file->off); - if (err) { - return err; + if (file->flags & LFS_F_INLINE) { + file->block = 0xfffffffe; + file->off = 0; + } else { + int err = lfs_ctz_find(lfs, &file->cache, NULL, + file->head, file->size, + file->pos, &file->block, &file->off); + if (err) { + return err; + } } file->flags |= LFS_F_READING; @@ -1761,31 +1817,42 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file, } while (nsize > 0) { + //printf("pos %d\n", file->pos + nsize); + // TODO combine with block allocation? + if (file->pos + nsize >= LFS_INLINE_MAX) { + file->flags &= ~LFS_F_INLINE; + } + // check if we need a new block if (!(file->flags & LFS_F_WRITING) || file->off == lfs->cfg->block_size) { - if (!(file->flags & LFS_F_WRITING) && file->pos > 0) { - // find out which block we're extending from - int err = lfs_ctz_find(lfs, &file->cache, NULL, - file->head, file->size, - file->pos-1, &file->block, &file->off); + if (file->flags & LFS_F_INLINE) { + file->block = 0xfffffffe; + file->off = 0; + } else { + if (!(file->flags & LFS_F_WRITING) && file->pos > 0) { + // find out which block we're extending from + int err = lfs_ctz_find(lfs, &file->cache, NULL, + file->head, file->size, + file->pos-1, &file->block, &file->off); + if (err) { + file->flags |= LFS_F_ERRED; + return err; + } + + // mark cache as dirty since we may have read data into it + file->cache.block = 0xffffffff; + } + + // extend file with new blocks + lfs_alloc_ack(lfs); + int err = lfs_ctz_extend(lfs, &lfs->rcache, &file->cache, + file->block, file->pos, + &file->block, &file->off); if (err) { file->flags |= LFS_F_ERRED; return err; } - - // mark cache as dirty since we may have read data into it - file->cache.block = 0xffffffff; - } - - // extend file with new blocks - lfs_alloc_ack(lfs); - int err = lfs_ctz_extend(lfs, &lfs->rcache, &file->cache, - file->block, file->pos, - &file->block, &file->off); - if (err) { - file->flags |= LFS_F_ERRED; - return err; } file->flags |= LFS_F_WRITING; diff --git a/lfs.h b/lfs.h index b35ad2492ba..75033b4cf53 100644 --- a/lfs.h +++ b/lfs.h @@ -55,6 +55,10 @@ typedef uint32_t lfs_block_t; #define LFS_NAME_MAX 255 #endif +#ifndef LFS_INLINE_MAX +#define LFS_INLINE_MAX 255 +#endif + // Possible error codes, these are negative to allow // valid positive return values enum lfs_error { @@ -82,25 +86,27 @@ enum lfs_type { // on disk structure LFS_STRUCT_CTZ = 0x10, LFS_STRUCT_DIR = 0x20, + LFS_STRUCT_INLINE = 0x30, LFS_STRUCT_MOVED = 0x80, }; // File open flags enum lfs_open_flags { // open flags - LFS_O_RDONLY = 1, // Open a file as read only - LFS_O_WRONLY = 2, // Open a file as write only - LFS_O_RDWR = 3, // Open a file as read and write - LFS_O_CREAT = 0x0100, // Create a file if it does not exist - LFS_O_EXCL = 0x0200, // Fail if a file already exists - LFS_O_TRUNC = 0x0400, // Truncate the existing file to zero size - LFS_O_APPEND = 0x0800, // Move to end of file on every write + LFS_O_RDONLY = 1, // Open a file as read only + LFS_O_WRONLY = 2, // Open a file as write only + LFS_O_RDWR = 3, // Open a file as read and write + LFS_O_CREAT = 0x0100, // Create a file if it does not exist + LFS_O_EXCL = 0x0200, // Fail if a file already exists + LFS_O_TRUNC = 0x0400, // Truncate the existing file to zero size + LFS_O_APPEND = 0x0800, // Move to end of file on every write // internally used flags - LFS_F_DIRTY = 0x10000, // File does not match storage - LFS_F_WRITING = 0x20000, // File has been written since last flush - LFS_F_READING = 0x40000, // File has been read since last flush - LFS_F_ERRED = 0x80000, // An error occured during write + LFS_F_DIRTY = 0x10000, // File does not match storage + LFS_F_WRITING = 0x20000, // File has been written since last flush + LFS_F_READING = 0x40000, // File has been read since last flush + LFS_F_ERRED = 0x80000, // An error occured during write + LFS_F_INLINE = 0x100000, // Currently inlined in directory entry }; // File seek flags diff --git a/tests/test_dirs.sh b/tests/test_dirs.sh index 53d76f7a8d2..11dc26ef971 100755 --- a/tests/test_dirs.sh +++ b/tests/test_dirs.sh @@ -357,7 +357,7 @@ tests/test.py << TEST TEST echo "--- Multi-block directory with files ---" -tests/test.py << TEST +tests/test.py -s << TEST lfs_mount(&lfs, &cfg) => 0; lfs_mkdir(&lfs, "prickly-pear") => 0; for (int i = 0; i < $LARGESIZE; i++) {