Merge pull request #1301 from bettio/posix-readdir

POSIX directory functions (opendir, closedir and readdir)

Directory listing using POSIX functions. Beware of 32 bit ARM and other 32 bit architectures, where
`ino_t ` is 32 bit while inodes are frequently 64 bit.

These changes are made under both the "Apache 2.0" and the "GNU Lesser General
Public License 2.1 or later" license terms (dual license).

SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later

.github/workflows/build-and-test-other.yaml

@@ -93,7 +93,9 @@ jobs:
         - arch: "arm32v7"
           platform: "arm/v7"
           tag: "bullseye"
-          cflags: "-mcpu=cortex-a7 -mfloat-abi=hard -O2 -mthumb -mthumb-interwork"
+          # -D_FILE_OFFSET_BITS=64 is required for making atomvm:posix_readdir/1 test work
+          # otherwise readdir will fail due to 64 bits inode numbers with 32 bit ino_t
+          cflags: "-mcpu=cortex-a7 -mfloat-abi=hard -O2 -mthumb -mthumb-interwork -D_FILE_OFFSET_BITS=64"
           cmake_opts: "-DAVM_WARNINGS_ARE_ERRORS=ON"
 
         - arch: "arm64v8"

CHANGELOG.md

@@ -34,6 +34,8 @@ also non string parameters (e.g. `Enum.join([1, 2], ",")`
 - Support for Elixir `IO.chardata_to_string/1`
 - Support for Elixir `List.duplicate/2`
 - Support for `binary:copy/1,2`
+- Support for directory listing using POSIX APIs: (`atomvm:posix_opendir/1`,
+`atomvm:posix_readdir/1`, `atomvm:posix_closedir/1`).
 
 ### Changed
 

libs/eavmlib/src/atomvm.erl

@@ -40,12 +40,16 @@
     posix_close/1,
     posix_read/2,
     posix_write/2,
-    posix_clock_settime/2
+    posix_clock_settime/2,
+    posix_opendir/1,
+    posix_closedir/1,
+    posix_readdir/1
 ]).
 
 -export_type([
     posix_fd/0,
-    posix_open_flag/0
+    posix_open_flag/0,
+    posix_dir/0
 ]).
 
 -deprecated([
@@ -84,6 +88,8 @@
     atom()
     | integer().
 
+-opaque posix_dir() :: binary().
+
 %%-----------------------------------------------------------------------------
 %% @returns The platform name.
 %% @doc     Return the platform moniker.
@@ -295,3 +301,37 @@ posix_write(_File, _Data) ->
     ok | {error, Reason :: posix_error()}.
 posix_clock_settime(_ClockId, _Time) ->
     erlang:nif_error(undefined).
+
+%%-----------------------------------------------------------------------------
+%% @param   Path    Path to the directory to open
+%% @returns A tuple with a directory descriptor or an error tuple.
+%% @doc     Open a file (on platforms that have `opendir(3)').
+%% @end
+%%-----------------------------------------------------------------------------
+-spec posix_opendir(Path :: iodata()) ->
+    {ok, posix_dir()} | {error, posix_error()}.
+posix_opendir(_Path) ->
+    erlang:nif_error(undefined).
+
+%%-----------------------------------------------------------------------------
+%% @param   Dir    Descriptor to a directory to close
+%% @returns `ok' or an error tuple
+%% @doc     Close a directory that was opened with `posix_opendir/1'
+%% @end
+%%-----------------------------------------------------------------------------
+-spec posix_closedir(Dir :: posix_dir()) -> ok | {error, posix_error()}.
+posix_closedir(_Dir) ->
+    erlang:nif_error(undefined).
+
+%%-----------------------------------------------------------------------------
+%% @param   Dir    Descriptor to an open directory
+%% @returns a `{dirent, InodeNo, Name}' tuple, `eof' or an error tuple
+%% @doc     Read a directory entry
+%% `eof' is returned if no more data can be read because the directory cursor
+%% reached the end.
+%% @end
+%%-----------------------------------------------------------------------------
+-spec posix_readdir(Dir :: posix_dir()) ->
+    {ok, {dirent, Inode :: integer(), Name :: binary()}} | eof | {error, posix_error()}.
+posix_readdir(_Dir) ->
+    erlang:nif_error(undefined).

src/libAtomVM/CMakeLists.txt

@@ -187,10 +187,13 @@ if (NOT ATOMIC_POINTER_LOCK_FREE_IS_TWO AND NOT (HAVE_PLATFORM_ATOMIC_H OR (AVM_
 endif()
 
 include(DefineIfExists)
-# HAVE_OPEN & HAVE_CLOSE are used in globalcontext.h
+# HAVE_OPEN, HAVE_OPENDIR, HAVE_CLOSE HAVE_CLOSEDIR, HAVE_READDIR are used in globalcontext.h
 define_if_function_exists(libAtomVM open "fcntl.h" PUBLIC HAVE_OPEN)
+define_if_function_exists(libAtomVM opendir "dirent.h" PUBLIC HAVE_OPENDIR)
 define_if_function_exists(libAtomVM close "unistd.h" PUBLIC HAVE_CLOSE)
+define_if_function_exists(libAtomVM closedir "dirent.h" PUBLIC HAVE_CLOSEDIR)
 define_if_function_exists(libAtomVM mkfifo "sys/stat.h" PRIVATE HAVE_MKFIFO)
+define_if_function_exists(libAtomVM readdir "dirent.h" PUBLIC HAVE_READDIR)
 define_if_function_exists(libAtomVM unlink "unistd.h" PRIVATE HAVE_UNLINK)
 define_if_symbol_exists(libAtomVM O_CLOEXEC "fcntl.h" PRIVATE HAVE_O_CLOEXEC)
 define_if_symbol_exists(libAtomVM O_DIRECTORY "fcntl.h" PRIVATE HAVE_O_DIRECTORY)

src/libAtomVM/globalcontext.c

@@ -135,6 +135,21 @@ GlobalContext *globalcontext_new()
     }
 #endif
 
+#if HAVE_OPENDIR && HAVE_READDIR && HAVE_CLOSEDIR
+    ErlNifEnv dir_env;
+    erl_nif_env_partial_init_from_globalcontext(&dir_env, glb);
+    glb->posix_dir_resource_type = enif_init_resource_type(&env, "posix_dir", &posix_dir_resource_type_init, ERL_NIF_RT_CREATE, NULL);
+    if (IS_NULL_PTR(glb->posix_dir_resource_type)) {
+#ifndef AVM_NO_SMP
+        smp_rwlock_destroy(glb->modules_lock);
+#endif
+        free(glb->modules_table);
+        atom_table_destroy(glb->atom_table);
+        free(glb);
+        return NULL;
+    }
+#endif
+
     sys_init_platform(glb);
 
 #ifndef AVM_NO_SMP

src/libAtomVM/globalcontext.h

@@ -152,6 +152,10 @@ struct GlobalContext
     ErlNifResourceType *posix_fd_resource_type;
 #endif
 
+#if HAVE_OPENDIR && HAVE_READDIR && HAVE_CLOSEDIR
+    ErlNifResourceType *posix_dir_resource_type;
+#endif
+
     void *platform_data;
 };
 

src/libAtomVM/nifs.c

@@ -794,6 +794,11 @@ DEFINE_MATH_NIF(tanh)
 #else
 #define IF_HAVE_CLOCK_SETTIME_OR_SETTIMEOFDAY(expr) NULL
 #endif
+#if HAVE_OPENDIR && HAVE_READDIR && HAVE_CLOSEDIR
+#define IF_HAVE_OPENDIR_READDIR_CLOSEDIR(expr) (expr)
+#else
+#define IF_HAVE_OPENDIR_READDIR_CLOSEDIR(expr) NULL
+#endif
 
 //Ignore warning caused by gperf generated code
 #pragma GCC diagnostic push

src/libAtomVM/nifs.gperf

@@ -141,6 +141,9 @@ atomvm:posix_select_stop/1, IF_HAVE_OPEN_CLOSE(&atomvm_posix_select_stop_nif)
 atomvm:posix_mkfifo/2, IF_HAVE_MKFIFO(&atomvm_posix_mkfifo_nif)
 atomvm:posix_unlink/1, IF_HAVE_UNLINK(&atomvm_posix_unlink_nif)
 atomvm:posix_clock_settime/2, IF_HAVE_CLOCK_SETTIME_OR_SETTIMEOFDAY(&atomvm_posix_clock_settime_nif)
+atomvm:posix_opendir/1, IF_HAVE_OPENDIR_READDIR_CLOSEDIR(&atomvm_posix_opendir_nif)
+atomvm:posix_closedir/1, IF_HAVE_OPENDIR_READDIR_CLOSEDIR(&atomvm_posix_closedir_nif)
+atomvm:posix_readdir/1, IF_HAVE_OPENDIR_READDIR_CLOSEDIR(&atomvm_posix_readdir_nif)
 code:load_abs/1, &code_load_abs_nif
 code:load_binary/3, &code_load_binary_nif
 code:ensure_loaded/1, &code_ensure_loaded_nif

src/libAtomVM/posix_nifs.c

@@ -38,10 +38,15 @@
 #include <sys/time.h>
 #endif
 
-#if HAVE_OPEN && HAVE_CLOSE || defined(HAVE_CLOCK_SETTIME) || defined(HAVE_SETTIMEOFDAY)
+#if HAVE_OPEN && HAVE_CLOSE || defined(HAVE_CLOCK_SETTIME) || defined(HAVE_SETTIMEOFDAY) \
+    || HAVE_OPENDIR && HAVE_READDIR && HAVE_CLOSEDIR
 #include <errno.h>
 #endif
 
+#if HAVE_OPENDIR && HAVE_READDIR && HAVE_CLOSEDIR
+#include <dirent.h>
+#endif
+
 #include "defaultatoms.h"
 #include "erl_nif_priv.h"
 #include "globalcontext.h"
@@ -602,6 +607,172 @@ static term nif_atomvm_posix_clock_settime(Context *ctx, int argc, term argv[])
 }
 #endif
 
+#if HAVE_OPENDIR && HAVE_READDIR && HAVE_CLOSEDIR
+struct PosixDir
+{
+    DIR *dir;
+};
+
+static void posix_dir_dtor(ErlNifEnv *caller_env, void *obj)
+{
+    UNUSED(caller_env);
+
+    struct PosixDir *dir_obj = (struct PosixDir *) obj;
+    if (dir_obj->dir) {
+        closedir(dir_obj->dir);
+        dir_obj->dir = NULL;
+    }
+}
+
+const ErlNifResourceTypeInit posix_dir_resource_type_init = {
+    .members = 1,
+    .dtor = posix_dir_dtor
+};
+
+static term errno_to_error_tuple_maybe_gc(Context *ctx)
+{
+    if (UNLIKELY(memory_ensure_free_opt(ctx, TUPLE_SIZE(2), MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) {
+        RAISE_ERROR(OUT_OF_MEMORY_ATOM);
+    }
+
+    term result = term_alloc_tuple(2, &ctx->heap);
+    term_put_tuple_element(result, 0, ERROR_ATOM);
+    term_put_tuple_element(result, 1, posix_errno_to_term(errno, ctx->global));
+
+    return result;
+}
+
+static term nif_atomvm_posix_opendir(Context *ctx, int argc, term argv[])
+{
+    UNUSED(argc);
+
+    GlobalContext *glb = ctx->global;
+
+    term path_term = argv[0];
+
+    int ok;
+    char *path = interop_term_to_string(path_term, &ok);
+    if (UNLIKELY(!ok)) {
+        RAISE_ERROR(BADARG_ATOM);
+    }
+
+    term result;
+    DIR *dir = opendir(path);
+    free(path);
+
+    if (IS_NULL_PTR(dir)) {
+        return errno_to_error_tuple_maybe_gc(ctx);
+    } else {
+        // Return a resource object
+        struct PosixDir *dir_obj
+            = enif_alloc_resource(glb->posix_dir_resource_type, sizeof(struct PosixDir));
+        if (IS_NULL_PTR(dir_obj)) {
+            RAISE_ERROR(OUT_OF_MEMORY_ATOM);
+        }
+        dir_obj->dir = dir;
+        if (UNLIKELY(memory_ensure_free_opt(
+                         ctx, TUPLE_SIZE(2) + TERM_BOXED_RESOURCE_SIZE, MEMORY_CAN_SHRINK)
+                != MEMORY_GC_OK)) {
+            RAISE_ERROR(OUT_OF_MEMORY_ATOM);
+        }
+        term obj = term_from_resource(dir_obj, &ctx->heap);
+        result = term_alloc_tuple(2, &ctx->heap);
+        term_put_tuple_element(result, 0, OK_ATOM);
+        term_put_tuple_element(result, 1, obj);
+    }
+
+    return result;
+}
+
+static term nif_atomvm_posix_closedir(Context *ctx, int argc, term argv[])
+{
+    UNUSED(argc);
+
+    term result = OK_ATOM;
+
+    void *dir_obj_ptr;
+    if (UNLIKELY(!enif_get_resource(erl_nif_env_from_context(ctx), argv[0],
+            ctx->global->posix_dir_resource_type, &dir_obj_ptr))) {
+        RAISE_ERROR(BADARG_ATOM);
+    }
+    struct PosixDir *dir_obj = (struct PosixDir *) dir_obj_ptr;
+    if (dir_obj->dir != NULL) {
+        if (UNLIKELY(closedir(dir_obj->dir) < 0)) {
+            dir_obj->dir = NULL; // even if bad things happen, do not close twice.
+            return errno_to_error_tuple_maybe_gc(ctx);
+        }
+        dir_obj->dir = NULL;
+    }
+
+    return result;
+}
+
+// This function main purpose is to avoid warnings, such as:
+// warning: comparison is always true due to limited range of data type [-Wtype-limits]
+static inline term to_boxed_safe(uint64_t value, Context *ctx)
+{
+    if (value <= INT64_MAX) {
+        return term_make_maybe_boxed_int64(value, &ctx->heap);
+    } else {
+        return UNDEFINED_ATOM;
+    }
+}
+
+static term nif_atomvm_posix_readdir(Context *ctx, int argc, term argv[])
+{
+    UNUSED(argc);
+
+    GlobalContext *glb = ctx->global;
+
+    void *dir_obj_ptr;
+    if (UNLIKELY(!enif_get_resource(
+            erl_nif_env_from_context(ctx), argv[0], glb->posix_dir_resource_type, &dir_obj_ptr))) {
+        RAISE_ERROR(BADARG_ATOM);
+    }
+    struct PosixDir *dir_obj = (struct PosixDir *) dir_obj_ptr;
+
+    errno = 0;
+    struct dirent *dir_result = readdir(dir_obj->dir);
+    if (dir_result == NULL) {
+        if (UNLIKELY(errno != 0)) {
+            return errno_to_error_tuple_maybe_gc(ctx);
+        }
+
+        return globalcontext_make_atom(glb, ATOM_STR("\x3", "eof"));
+    }
+
+    size_t name_len = strlen(dir_result->d_name);
+    if (UNLIKELY(
+            memory_ensure_free_opt(ctx,
+                BOXED_INT64_SIZE + term_binary_heap_size(name_len) + TUPLE_SIZE(3) + TUPLE_SIZE(2),
+                MEMORY_CAN_SHRINK)
+            != MEMORY_GC_OK)) {
+        RAISE_ERROR(OUT_OF_MEMORY_ATOM);
+    }
+
+    term ino_no = to_boxed_safe(dir_result->d_ino, ctx);
+
+    term name_term = term_create_uninitialized_binary(name_len, &ctx->heap, glb);
+    memcpy((void *) term_binary_data(name_term), dir_result->d_name, name_len);
+
+    term dirent_atom = globalcontext_make_atom(glb, ATOM_STR("\x6", "dirent"));
+
+    // {dirent, Inode, Name}
+    term dirent_term = term_alloc_tuple(3, &ctx->heap);
+    term_put_tuple_element(dirent_term, 0, dirent_atom);
+    term_put_tuple_element(dirent_term, 1, ino_no);
+    term_put_tuple_element(dirent_term, 2, name_term);
+
+    // {ok, DirentTuple}
+    term result = term_alloc_tuple(2, &ctx->heap);
+    term_put_tuple_element(result, 0, OK_ATOM);
+    term_put_tuple_element(result, 1, dirent_term);
+
+    return result;
+}
+
+#endif
+
 #if HAVE_OPEN && HAVE_CLOSE
 const struct Nif atomvm_posix_open_nif = {
     .base.type = NIFFunctionType,
@@ -650,3 +821,17 @@ const struct Nif atomvm_posix_clock_settime_nif = {
     .nif_ptr = nif_atomvm_posix_clock_settime
 };
 #endif
+#if HAVE_OPENDIR && HAVE_READDIR && HAVE_CLOSEDIR
+const struct Nif atomvm_posix_opendir_nif = {
+    .base.type = NIFFunctionType,
+    .nif_ptr = nif_atomvm_posix_opendir
+};
+const struct Nif atomvm_posix_closedir_nif = {
+    .base.type = NIFFunctionType,
+    .nif_ptr = nif_atomvm_posix_closedir
+};
+const struct Nif atomvm_posix_readdir_nif = {
+    .base.type = NIFFunctionType,
+    .nif_ptr = nif_atomvm_posix_readdir
+};
+#endif

src/libAtomVM/posix_nifs.h

@@ -53,6 +53,12 @@ extern const struct Nif atomvm_posix_unlink_nif;
 #if defined(HAVE_CLOCK_SETTIME) || defined(HAVE_SETTIMEOFDAY)
 extern const struct Nif atomvm_posix_clock_settime_nif;
 #endif
+#if HAVE_OPENDIR && HAVE_READDIR && HAVE_CLOSEDIR
+extern const ErlNifResourceTypeInit posix_dir_resource_type_init;
+extern const struct Nif atomvm_posix_opendir_nif;
+extern const struct Nif atomvm_posix_readdir_nif;
+extern const struct Nif atomvm_posix_closedir_nif;
+#endif
 
 /**
  * @brief Convenient function to return posix errors as atom.

src/platforms/esp32/CMakeLists.txt

@@ -31,6 +31,12 @@ set(HAVE_MKFIFO "" CACHE INTERNAL "Have symbol mkfifo" FORCE)
 # in CMAKE_REQUIRED_INCLUDES as lwip includes freetos and many esp system components
 set(HAVE_SOCKET 1 CACHE INTERNAL "Have symbol socket" FORCE)
 
+# opendir, closedir and readdir functions are not detected
+# but they are available
+set(HAVE_OPENDIR 1 CACHE INTERNAL "Have symbol opendir" FORCE)
+set(HAVE_CLOSEDIR 1 CACHE INTERNAL "Have symbol closedir" FORCE)
+set(HAVE_READDIR 1 CACHE INTERNAL "Have symbol readdir" FORCE)
+
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../../CMakeModules")
 
 # Disable SMP with esp32 socs that have only one core

tests/libs/eavmlib/CMakeLists.txt

@@ -23,6 +23,7 @@ project(test_eavmlib)
 include(BuildErlang)
 
 set(ERLANG_MODULES
+    test_dir
     test_file
     test_ahttp_client
     test_port