Skip to content

Commit

Permalink
Shoehorned in hacky implementation of inline files
Browse files Browse the repository at this point in the history 
Proof-of-concept implementation of inline files that stores the file's
content directly in its parent's directory pair.

Inline files are indicated by a different type stored in an entry's
struct field, and take advantage of resizable entries. Where a normal
file's entry would normally hold the reference to the CTZ skip-list, an
inline file's entry contains the contents of the actual file.

Unfortunately, storing the inline file on disk is the easy part. We also
need to manage inline files in the internals of littlefs and provide the
same operations that we do on normal files, all while reusing as much
code as possible to avoid a significant increase in code cost.

There is a relatively simple, though maybe a bit hacky, solution here. If a
file fits entirely in a cache line, the file logic never actually has to go to
disk. This means we can just give the file a "pretend" block (hopefully
one that would assert if ever written to), and carry out file operations
as normal, as long as we catch the file before it exceeds the cache line
and write out the file to an actual disk.
  • Loading branch information
@geky
geky committed Oct 10, 2018
1 parent fb23044 commit 836e238
Show file tree
Hide file tree
Showing 3 changed files with 133 additions and 60 deletions.
163 changes: 115 additions & 48 deletions lfs.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ static int lfs_cache_read(lfs_t *lfs, lfs_cache_t *rcache,
const lfs_cache_t *pcache, lfs_block_t block,
lfs_off_t off, void *buffer, lfs_size_t size) {
uint8_t *data = buffer;
LFS_ASSERT(block < lfs->cfg->block_count);
LFS_ASSERT(block != 0xffffffff);

while (size > 0) {
if (pcache && block == pcache->block && off >= pcache->off &&
Expand Down Expand Up @@ -68,6 +68,7 @@ static int lfs_cache_read(lfs_t *lfs, lfs_cache_t *rcache,
}

// load to cache, first condition can no longer fail
LFS_ASSERT(block < lfs->cfg->block_count);
rcache->block = block;
rcache->off = off - (off % lfs->cfg->read_size);
int err = lfs->cfg->read(lfs->cfg, rcache->block,
Expand Down Expand Up @@ -121,6 +122,7 @@ static int lfs_cache_crc(lfs_t *lfs, lfs_cache_t *rcache,
static int lfs_cache_flush(lfs_t *lfs,
lfs_cache_t *pcache, lfs_cache_t *rcache) {
if (pcache->block != 0xffffffff) {
LFS_ASSERT(pcache->block < lfs->cfg->block_count);
int err = lfs->cfg->prog(lfs->cfg, pcache->block,
pcache->off, pcache->buffer, lfs->cfg->prog_size);
if (err) {
Expand Down Expand Up @@ -149,7 +151,7 @@ static int lfs_cache_prog(lfs_t *lfs, lfs_cache_t *pcache,
lfs_cache_t *rcache, lfs_block_t block,
lfs_off_t off, const void *buffer, lfs_size_t size) {
const uint8_t *data = buffer;
LFS_ASSERT(block < lfs->cfg->block_count);
LFS_ASSERT(block != 0xffffffff);

while (size > 0) {
if (block == pcache->block && off >= pcache->off &&
Expand Down Expand Up @@ -181,6 +183,7 @@ static int lfs_cache_prog(lfs_t *lfs, lfs_cache_t *pcache,
if (off % lfs->cfg->prog_size == 0 &&
size >= lfs->cfg->prog_size) {
// bypass pcache?
LFS_ASSERT(block < lfs->cfg->block_count);
lfs_size_t diff = size - (size % lfs->cfg->prog_size);
int err = lfs->cfg->prog(lfs->cfg, block, off, data, diff);
if (err) {
Expand Down Expand Up @@ -240,6 +243,7 @@ static int lfs_bd_crc(lfs_t *lfs, lfs_block_t block,
}

static int lfs_bd_erase(lfs_t *lfs, lfs_block_t block) {
LFS_ASSERT(block < lfs->cfg->block_count);
return lfs->cfg->erase(lfs->cfg, block);
}

Expand Down Expand Up @@ -851,7 +855,7 @@ static int lfs_dir_update(lfs_t *lfs, lfs_dir_t *dir,
}

static int lfs_dir_next(lfs_t *lfs, lfs_dir_t *dir, lfs_entry_t *entry) {
while (dir->off + sizeof(entry->d) > (0x7fffffff & dir->d.size)-4) {
while (dir->off >= (0x7fffffff & dir->d.size)-4) {
if (!(0x80000000 & dir->d.size)) {
entry->off = dir->off;
return LFS_ERR_NOENT;
Expand Down Expand Up @@ -942,8 +946,8 @@ static int lfs_dir_find(lfs_t *lfs, lfs_dir_t *dir,
return err;
}

if (((0x7f & entry->d.type) != (LFS_STRUCT_CTZ | LFS_TYPE_REG) &&
(0x7f & entry->d.type) != (LFS_STRUCT_DIR | LFS_TYPE_DIR)) ||
if (((0xf & entry->d.type) != LFS_TYPE_REG &&
(0xf & entry->d.type) != LFS_TYPE_DIR) ||
entry->d.nlen != pathlen) {
continue;
}
Expand Down Expand Up @@ -1126,8 +1130,8 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) {
return (err == LFS_ERR_NOENT) ? 0 : err;
}

if ((0x7f & entry.d.type) != (LFS_STRUCT_CTZ | LFS_TYPE_REG) &&
(0x7f & entry.d.type) != (LFS_STRUCT_DIR | LFS_TYPE_DIR)) {
if ((0xf & entry.d.type) != LFS_TYPE_REG &&
(0xf & entry.d.type) != LFS_TYPE_DIR) {
continue;
}

Expand All @@ -1149,8 +1153,10 @@ int lfs_dir_read(lfs_t *lfs, lfs_dir_t *dir, struct lfs_info *info) {
}

info->type = 0xf & entry.d.type;
if (info->type == LFS_TYPE_REG) {
if (entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG)) {
info->size = entry.d.u.file.size;
} else if (entry.d.type == (LFS_STRUCT_INLINE | LFS_TYPE_REG)) {
info->size = entry.d.elen;
}

int err = lfs_bd_read(lfs, dir->pair[0],
Expand Down Expand Up @@ -1424,18 +1430,16 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file,
}

// create entry to remember name
entry.d.type = LFS_STRUCT_CTZ | LFS_TYPE_REG;
entry.d.elen = sizeof(entry.d) - 4;
entry.d.type = LFS_STRUCT_INLINE | LFS_TYPE_REG;
entry.d.elen = 0;
entry.d.alen = 0;
entry.d.nlen = strlen(path);
entry.d.u.file.head = 0xffffffff;
entry.d.u.file.size = 0;
entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen;

err = lfs_dir_append(lfs, &cwd, &entry,
&(struct lfs_region){
0, +sizeof(entry.d),
lfs_commit_mem, &entry.d, sizeof(entry.d),
0, +4,
lfs_commit_mem, &entry.d, 4,
&(struct lfs_region){
0, +entry.d.nlen,
lfs_commit_mem, path, entry.d.nlen}});
Expand Down Expand Up @@ -1481,6 +1485,22 @@ int lfs_file_open(lfs_t *lfs, lfs_file_t *file,
}
}

// load inline files
if ((0x70 & entry.d.type) == LFS_STRUCT_INLINE) {
file->head = 0xfffffffe;
file->size = entry.d.elen;
file->flags |= LFS_F_INLINE;
file->cache.block = file->head;
file->cache.off = 0;
err = lfs_bd_read(lfs, cwd.pair[0],
entry.off + 4,
file->cache.buffer, file->size);
if (err) {
lfs_free(file->cache.buffer);
return err;
}
}

// add to list of files
file->next = lfs->files;
lfs->files = file;
Expand Down Expand Up @@ -1556,6 +1576,20 @@ static int lfs_file_relocate(lfs_t *lfs, lfs_file_t *file) {
}

static int lfs_file_flush(lfs_t *lfs, lfs_file_t *file) {
if (file->flags & LFS_F_INLINE) {
// do nothing since we won't need the cache for anything else
if (file->flags & LFS_F_READING) {
file->flags &= ~LFS_F_READING;
}

if (file->flags & LFS_F_WRITING) {
file->flags &= ~LFS_F_WRITING;
file->flags |= LFS_F_DIRTY;
}

return 0;
}

if (file->flags & LFS_F_READING) {
// just drop read cache
file->cache.block = 0xffffffff;
Expand Down Expand Up @@ -1642,26 +1676,43 @@ int lfs_file_sync(lfs_t *lfs, lfs_file_t *file) {
return err;
}

// TODO entry read?
lfs_entry_t entry = {.off = file->poff};
err = lfs_bd_read(lfs, cwd.pair[0], entry.off,
&entry.d, sizeof(entry.d));
lfs_entry_fromle32(&entry.d);
if (err) {
return err;
}
LFS_ASSERT((0xf & entry.d.type) == LFS_TYPE_REG);
entry.size = 4 + entry.d.elen + entry.d.alen + entry.d.nlen;

LFS_ASSERT(entry.d.type == (LFS_STRUCT_CTZ | LFS_TYPE_REG));
entry.d.u.file.head = file->head;
entry.d.u.file.size = file->size;
if (file->flags & LFS_F_INLINE) {
file->size = lfs_max(file->pos, file->size);
lfs_ssize_t diff = file->size - entry.d.elen;
entry.d.elen = file->size;

lfs_entry_tole32(&entry.d);
err = lfs_dir_update(lfs, &cwd, &entry,
&(struct lfs_region){
0, 0,
lfs_commit_mem, &entry.d, sizeof(entry.d)});
lfs_entry_fromle32(&entry.d);
if (err) {
return err;
err = lfs_dir_update(lfs, &cwd, &entry,
&(struct lfs_region){
0, 0,
lfs_commit_mem, &entry.d, 4,
&(struct lfs_region){
4, diff,
lfs_commit_mem, file->cache.buffer, file->size}});
if (err) {
return err;
}
} else {
entry.d.u.file.head = file->head;
entry.d.u.file.size = file->size;

err = lfs_dir_update(lfs, &cwd, &entry,
&(struct lfs_region){
0, 0,
lfs_commit_mem, &entry.d, sizeof(entry.d)});
if (err) {
return err;
}
}

file->flags &= ~LFS_F_DIRTY;
Expand Down Expand Up @@ -1699,11 +1750,16 @@ lfs_ssize_t lfs_file_read(lfs_t *lfs, lfs_file_t *file,
// check if we need a new block
if (!(file->flags & LFS_F_READING) ||
file->off == lfs->cfg->block_size) {
int err = lfs_ctz_find(lfs, &file->cache, NULL,
file->head, file->size,
file->pos, &file->block, &file->off);
if (err) {
return err;
if (file->flags & LFS_F_INLINE) {
file->block = 0xfffffffe;
file->off = 0;
} else {
int err = lfs_ctz_find(lfs, &file->cache, NULL,
file->head, file->size,
file->pos, &file->block, &file->off);
if (err) {
return err;
}
}

file->flags |= LFS_F_READING;
Expand Down Expand Up @@ -1761,31 +1817,42 @@ lfs_ssize_t lfs_file_write(lfs_t *lfs, lfs_file_t *file,
}

while (nsize > 0) {
//printf("pos %d\n", file->pos + nsize);
// TODO combine with block allocation?
if (file->pos + nsize >= LFS_INLINE_MAX) {
file->flags &= ~LFS_F_INLINE;
}

// check if we need a new block
if (!(file->flags & LFS_F_WRITING) ||
file->off == lfs->cfg->block_size) {
if (!(file->flags & LFS_F_WRITING) && file->pos > 0) {
// find out which block we're extending from
int err = lfs_ctz_find(lfs, &file->cache, NULL,
file->head, file->size,
file->pos-1, &file->block, &file->off);
if (file->flags & LFS_F_INLINE) {
file->block = 0xfffffffe;
file->off = 0;
} else {
if (!(file->flags & LFS_F_WRITING) && file->pos > 0) {
// find out which block we're extending from
int err = lfs_ctz_find(lfs, &file->cache, NULL,
file->head, file->size,
file->pos-1, &file->block, &file->off);
if (err) {
file->flags |= LFS_F_ERRED;
return err;
}

// mark cache as dirty since we may have read data into it
file->cache.block = 0xffffffff;
}

// extend file with new blocks
lfs_alloc_ack(lfs);
int err = lfs_ctz_extend(lfs, &lfs->rcache, &file->cache,
file->block, file->pos,
&file->block, &file->off);
if (err) {
file->flags |= LFS_F_ERRED;
return err;
}

// mark cache as dirty since we may have read data into it
file->cache.block = 0xffffffff;
}

// extend file with new blocks
lfs_alloc_ack(lfs);
int err = lfs_ctz_extend(lfs, &lfs->rcache, &file->cache,
file->block, file->pos,
&file->block, &file->off);
if (err) {
file->flags |= LFS_F_ERRED;
return err;
}

file->flags |= LFS_F_WRITING;
Expand Down
28 changes: 17 additions & 11 deletions lfs.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,10 @@ typedef uint32_t lfs_block_t;
#define LFS_NAME_MAX 255
#endif

#ifndef LFS_INLINE_MAX
#define LFS_INLINE_MAX 255
#endif

// Possible error codes, these are negative to allow
// valid positive return values
enum lfs_error {
Expand Down Expand Up @@ -82,25 +86,27 @@ enum lfs_type {
// on disk structure
LFS_STRUCT_CTZ = 0x10,
LFS_STRUCT_DIR = 0x20,
LFS_STRUCT_INLINE = 0x30,
LFS_STRUCT_MOVED = 0x80,
};

// File open flags
enum lfs_open_flags {
// open flags
LFS_O_RDONLY = 1, // Open a file as read only
LFS_O_WRONLY = 2, // Open a file as write only
LFS_O_RDWR = 3, // Open a file as read and write
LFS_O_CREAT = 0x0100, // Create a file if it does not exist
LFS_O_EXCL = 0x0200, // Fail if a file already exists
LFS_O_TRUNC = 0x0400, // Truncate the existing file to zero size
LFS_O_APPEND = 0x0800, // Move to end of file on every write
LFS_O_RDONLY = 1, // Open a file as read only
LFS_O_WRONLY = 2, // Open a file as write only
LFS_O_RDWR = 3, // Open a file as read and write
LFS_O_CREAT = 0x0100, // Create a file if it does not exist
LFS_O_EXCL = 0x0200, // Fail if a file already exists
LFS_O_TRUNC = 0x0400, // Truncate the existing file to zero size
LFS_O_APPEND = 0x0800, // Move to end of file on every write

// internally used flags
LFS_F_DIRTY = 0x10000, // File does not match storage
LFS_F_WRITING = 0x20000, // File has been written since last flush
LFS_F_READING = 0x40000, // File has been read since last flush
LFS_F_ERRED = 0x80000, // An error occured during write
LFS_F_DIRTY = 0x10000, // File does not match storage
LFS_F_WRITING = 0x20000, // File has been written since last flush
LFS_F_READING = 0x40000, // File has been read since last flush
LFS_F_ERRED = 0x80000, // An error occured during write
LFS_F_INLINE = 0x100000, // Currently inlined in directory entry
};

// File seek flags
Expand Down
2 changes: 1 addition & 1 deletion tests/test_dirs.sh
View file
Original file line number Diff line number Diff line change
Expand Up @@ -357,7 +357,7 @@ tests/test.py << TEST
TEST

echo "--- Multi-block directory with files ---"
tests/test.py << TEST
tests/test.py -s << TEST
lfs_mount(&lfs, &cfg) => 0;
lfs_mkdir(&lfs, "prickly-pear") => 0;
for (int i = 0; i < $LARGESIZE; i++) {
Expand Down

0 comments on commit 836e238

Please sign in to comment.