From e8af06a42970617eb6952ccc3899fdbe9adf9b94 Mon Sep 17 00:00:00 2001 From: Davide Bettio Date: Mon, 30 Sep 2024 01:23:08 +0200 Subject: [PATCH] Add POSIX functions to list directory entries on supported platforms Add `atomvm:posix_opendir/1`, `atomvm:posix_closedir/1` and `atomvm:posix_readdir/1` functions. Signed-off-by: Davide Bettio --- .github/workflows/build-and-test-other.yaml | 4 +- CHANGELOG.md | 2 + libs/eavmlib/src/atomvm.erl | 44 ++++- src/libAtomVM/CMakeLists.txt | 5 +- src/libAtomVM/globalcontext.c | 15 ++ src/libAtomVM/globalcontext.h | 4 + src/libAtomVM/nifs.c | 5 + src/libAtomVM/nifs.gperf | 3 + src/libAtomVM/posix_nifs.c | 187 +++++++++++++++++++- src/libAtomVM/posix_nifs.h | 6 + src/platforms/esp32/CMakeLists.txt | 6 + tests/libs/eavmlib/CMakeLists.txt | 1 + tests/libs/eavmlib/test_dir.erl | 38 ++++ tests/libs/eavmlib/tests.erl | 1 + 14 files changed, 316 insertions(+), 5 deletions(-) create mode 100644 tests/libs/eavmlib/test_dir.erl diff --git a/.github/workflows/build-and-test-other.yaml b/.github/workflows/build-and-test-other.yaml index 49b18c44c..98502a79d 100644 --- a/.github/workflows/build-and-test-other.yaml +++ b/.github/workflows/build-and-test-other.yaml @@ -93,7 +93,9 @@ jobs: - arch: "arm32v7" platform: "arm/v7" tag: "bullseye" - cflags: "-mcpu=cortex-a7 -mfloat-abi=hard -O2 -mthumb -mthumb-interwork" + # -D_FILE_OFFSET_BITS=64 is required for making atomvm:posix_readdir/1 test work + # otherwise readdir will fail due to 64 bits inode numbers with 32 bit ino_t + cflags: "-mcpu=cortex-a7 -mfloat-abi=hard -O2 -mthumb -mthumb-interwork -D_FILE_OFFSET_BITS=64" cmake_opts: "-DAVM_WARNINGS_ARE_ERRORS=ON" - arch: "arm64v8" diff --git a/CHANGELOG.md b/CHANGELOG.md index d0ec7173d..3bafb392f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -34,6 +34,8 @@ also non string parameters (e.g. `Enum.join([1, 2], ",")` - Support for Elixir `IO.chardata_to_string/1` - Support for Elixir `List.duplicate/2` - Support for `binary:copy/1,2` +- Support for directory listing using POSIX APIs: (`atomvm:posix_opendir/1`, +`atomvm:posix_readdir/1`, `atomvm:posix_closedir/1`). ### Changed diff --git a/libs/eavmlib/src/atomvm.erl b/libs/eavmlib/src/atomvm.erl index e9cd15e1c..a5dd82645 100644 --- a/libs/eavmlib/src/atomvm.erl +++ b/libs/eavmlib/src/atomvm.erl @@ -40,12 +40,16 @@ posix_close/1, posix_read/2, posix_write/2, - posix_clock_settime/2 + posix_clock_settime/2, + posix_opendir/1, + posix_closedir/1, + posix_readdir/1 ]). -export_type([ posix_fd/0, - posix_open_flag/0 + posix_open_flag/0, + posix_dir/0 ]). -deprecated([ @@ -84,6 +88,8 @@ atom() | integer(). +-opaque posix_dir() :: binary(). + %%----------------------------------------------------------------------------- %% @returns The platform name. %% @doc Return the platform moniker. @@ -295,3 +301,37 @@ posix_write(_File, _Data) -> ok | {error, Reason :: posix_error()}. posix_clock_settime(_ClockId, _Time) -> erlang:nif_error(undefined). + +%%----------------------------------------------------------------------------- +%% @param Path Path to the directory to open +%% @returns A tuple with a directory descriptor or an error tuple. +%% @doc Open a file (on platforms that have `opendir(3)'). +%% @end +%%----------------------------------------------------------------------------- +-spec posix_opendir(Path :: iodata()) -> + {ok, posix_dir()} | {error, posix_error()}. +posix_opendir(_Path) -> + erlang:nif_error(undefined). + +%%----------------------------------------------------------------------------- +%% @param Dir Descriptor to a directory to close +%% @returns `ok' or an error tuple +%% @doc Close a directory that was opened with `posix_opendir/1' +%% @end +%%----------------------------------------------------------------------------- +-spec posix_closedir(Dir :: posix_dir()) -> ok | {error, posix_error()}. +posix_closedir(_Dir) -> + erlang:nif_error(undefined). + +%%----------------------------------------------------------------------------- +%% @param Dir Descriptor to an open directory +%% @returns a `{dirent, InodeNo, Name}' tuple, `eof' or an error tuple +%% @doc Read a directory entry +%% `eof' is returned if no more data can be read because the directory cursor +%% reached the end. +%% @end +%%----------------------------------------------------------------------------- +-spec posix_readdir(Dir :: posix_dir()) -> + {ok, {dirent, Inode :: integer(), Name :: binary()}} | eof | {error, posix_error()}. +posix_readdir(_Dir) -> + erlang:nif_error(undefined). diff --git a/src/libAtomVM/CMakeLists.txt b/src/libAtomVM/CMakeLists.txt index c4273607e..d96503a04 100644 --- a/src/libAtomVM/CMakeLists.txt +++ b/src/libAtomVM/CMakeLists.txt @@ -187,10 +187,13 @@ if (NOT ATOMIC_POINTER_LOCK_FREE_IS_TWO AND NOT (HAVE_PLATFORM_ATOMIC_H OR (AVM_ endif() include(DefineIfExists) -# HAVE_OPEN & HAVE_CLOSE are used in globalcontext.h +# HAVE_OPEN, HAVE_OPENDIR, HAVE_CLOSE HAVE_CLOSEDIR, HAVE_READDIR are used in globalcontext.h define_if_function_exists(libAtomVM open "fcntl.h" PUBLIC HAVE_OPEN) +define_if_function_exists(libAtomVM opendir "dirent.h" PUBLIC HAVE_OPENDIR) define_if_function_exists(libAtomVM close "unistd.h" PUBLIC HAVE_CLOSE) +define_if_function_exists(libAtomVM closedir "dirent.h" PUBLIC HAVE_CLOSEDIR) define_if_function_exists(libAtomVM mkfifo "sys/stat.h" PRIVATE HAVE_MKFIFO) +define_if_function_exists(libAtomVM readdir "dirent.h" PUBLIC HAVE_READDIR) define_if_function_exists(libAtomVM unlink "unistd.h" PRIVATE HAVE_UNLINK) define_if_symbol_exists(libAtomVM O_CLOEXEC "fcntl.h" PRIVATE HAVE_O_CLOEXEC) define_if_symbol_exists(libAtomVM O_DIRECTORY "fcntl.h" PRIVATE HAVE_O_DIRECTORY) diff --git a/src/libAtomVM/globalcontext.c b/src/libAtomVM/globalcontext.c index fc28c4ead..54a8d291e 100644 --- a/src/libAtomVM/globalcontext.c +++ b/src/libAtomVM/globalcontext.c @@ -135,6 +135,21 @@ GlobalContext *globalcontext_new() } #endif +#if HAVE_OPENDIR && HAVE_READDIR && HAVE_CLOSEDIR + ErlNifEnv dir_env; + erl_nif_env_partial_init_from_globalcontext(&dir_env, glb); + glb->posix_dir_resource_type = enif_init_resource_type(&env, "posix_dir", &posix_dir_resource_type_init, ERL_NIF_RT_CREATE, NULL); + if (IS_NULL_PTR(glb->posix_dir_resource_type)) { +#ifndef AVM_NO_SMP + smp_rwlock_destroy(glb->modules_lock); +#endif + free(glb->modules_table); + atom_table_destroy(glb->atom_table); + free(glb); + return NULL; + } +#endif + sys_init_platform(glb); #ifndef AVM_NO_SMP diff --git a/src/libAtomVM/globalcontext.h b/src/libAtomVM/globalcontext.h index 605600618..71f88b633 100644 --- a/src/libAtomVM/globalcontext.h +++ b/src/libAtomVM/globalcontext.h @@ -152,6 +152,10 @@ struct GlobalContext ErlNifResourceType *posix_fd_resource_type; #endif +#if HAVE_OPENDIR && HAVE_READDIR && HAVE_CLOSEDIR + ErlNifResourceType *posix_dir_resource_type; +#endif + void *platform_data; }; diff --git a/src/libAtomVM/nifs.c b/src/libAtomVM/nifs.c index 03daba1a7..798ba07b7 100644 --- a/src/libAtomVM/nifs.c +++ b/src/libAtomVM/nifs.c @@ -794,6 +794,11 @@ DEFINE_MATH_NIF(tanh) #else #define IF_HAVE_CLOCK_SETTIME_OR_SETTIMEOFDAY(expr) NULL #endif +#if HAVE_OPENDIR && HAVE_READDIR && HAVE_CLOSEDIR +#define IF_HAVE_OPENDIR_READDIR_CLOSEDIR(expr) (expr) +#else +#define IF_HAVE_OPENDIR_READDIR_CLOSEDIR(expr) NULL +#endif //Ignore warning caused by gperf generated code #pragma GCC diagnostic push diff --git a/src/libAtomVM/nifs.gperf b/src/libAtomVM/nifs.gperf index f23644630..e3a9e8246 100644 --- a/src/libAtomVM/nifs.gperf +++ b/src/libAtomVM/nifs.gperf @@ -141,6 +141,9 @@ atomvm:posix_select_stop/1, IF_HAVE_OPEN_CLOSE(&atomvm_posix_select_stop_nif) atomvm:posix_mkfifo/2, IF_HAVE_MKFIFO(&atomvm_posix_mkfifo_nif) atomvm:posix_unlink/1, IF_HAVE_UNLINK(&atomvm_posix_unlink_nif) atomvm:posix_clock_settime/2, IF_HAVE_CLOCK_SETTIME_OR_SETTIMEOFDAY(&atomvm_posix_clock_settime_nif) +atomvm:posix_opendir/1, IF_HAVE_OPENDIR_READDIR_CLOSEDIR(&atomvm_posix_opendir_nif) +atomvm:posix_closedir/1, IF_HAVE_OPENDIR_READDIR_CLOSEDIR(&atomvm_posix_closedir_nif) +atomvm:posix_readdir/1, IF_HAVE_OPENDIR_READDIR_CLOSEDIR(&atomvm_posix_readdir_nif) code:load_abs/1, &code_load_abs_nif code:load_binary/3, &code_load_binary_nif code:ensure_loaded/1, &code_ensure_loaded_nif diff --git a/src/libAtomVM/posix_nifs.c b/src/libAtomVM/posix_nifs.c index 07881e0ff..ceeb9f854 100644 --- a/src/libAtomVM/posix_nifs.c +++ b/src/libAtomVM/posix_nifs.c @@ -38,10 +38,15 @@ #include #endif -#if HAVE_OPEN && HAVE_CLOSE || defined(HAVE_CLOCK_SETTIME) || defined(HAVE_SETTIMEOFDAY) +#if HAVE_OPEN && HAVE_CLOSE || defined(HAVE_CLOCK_SETTIME) || defined(HAVE_SETTIMEOFDAY) \ + || HAVE_OPENDIR && HAVE_READDIR && HAVE_CLOSEDIR #include #endif +#if HAVE_OPENDIR && HAVE_READDIR && HAVE_CLOSEDIR +#include +#endif + #include "defaultatoms.h" #include "erl_nif_priv.h" #include "globalcontext.h" @@ -602,6 +607,172 @@ static term nif_atomvm_posix_clock_settime(Context *ctx, int argc, term argv[]) } #endif +#if HAVE_OPENDIR && HAVE_READDIR && HAVE_CLOSEDIR +struct PosixDir +{ + DIR *dir; +}; + +static void posix_dir_dtor(ErlNifEnv *caller_env, void *obj) +{ + UNUSED(caller_env); + + struct PosixDir *dir_obj = (struct PosixDir *) obj; + if (dir_obj->dir) { + closedir(dir_obj->dir); + dir_obj->dir = NULL; + } +} + +const ErlNifResourceTypeInit posix_dir_resource_type_init = { + .members = 1, + .dtor = posix_dir_dtor +}; + +static term errno_to_error_tuple_maybe_gc(Context *ctx) +{ + if (UNLIKELY(memory_ensure_free_opt(ctx, TUPLE_SIZE(2), MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { + RAISE_ERROR(OUT_OF_MEMORY_ATOM); + } + + term result = term_alloc_tuple(2, &ctx->heap); + term_put_tuple_element(result, 0, ERROR_ATOM); + term_put_tuple_element(result, 1, posix_errno_to_term(errno, ctx->global)); + + return result; +} + +static term nif_atomvm_posix_opendir(Context *ctx, int argc, term argv[]) +{ + UNUSED(argc); + + GlobalContext *glb = ctx->global; + + term path_term = argv[0]; + + int ok; + char *path = interop_term_to_string(path_term, &ok); + if (UNLIKELY(!ok)) { + RAISE_ERROR(BADARG_ATOM); + } + + term result; + DIR *dir = opendir(path); + free(path); + + if (IS_NULL_PTR(dir)) { + return errno_to_error_tuple_maybe_gc(ctx); + } else { + // Return a resource object + struct PosixDir *dir_obj + = enif_alloc_resource(glb->posix_dir_resource_type, sizeof(struct PosixDir)); + if (IS_NULL_PTR(dir_obj)) { + RAISE_ERROR(OUT_OF_MEMORY_ATOM); + } + dir_obj->dir = dir; + if (UNLIKELY(memory_ensure_free_opt( + ctx, TUPLE_SIZE(2) + TERM_BOXED_RESOURCE_SIZE, MEMORY_CAN_SHRINK) + != MEMORY_GC_OK)) { + RAISE_ERROR(OUT_OF_MEMORY_ATOM); + } + term obj = term_from_resource(dir_obj, &ctx->heap); + result = term_alloc_tuple(2, &ctx->heap); + term_put_tuple_element(result, 0, OK_ATOM); + term_put_tuple_element(result, 1, obj); + } + + return result; +} + +static term nif_atomvm_posix_closedir(Context *ctx, int argc, term argv[]) +{ + UNUSED(argc); + + term result = OK_ATOM; + + void *dir_obj_ptr; + if (UNLIKELY(!enif_get_resource(erl_nif_env_from_context(ctx), argv[0], + ctx->global->posix_dir_resource_type, &dir_obj_ptr))) { + RAISE_ERROR(BADARG_ATOM); + } + struct PosixDir *dir_obj = (struct PosixDir *) dir_obj_ptr; + if (dir_obj->dir != NULL) { + if (UNLIKELY(closedir(dir_obj->dir) < 0)) { + dir_obj->dir = NULL; // even if bad things happen, do not close twice. + return errno_to_error_tuple_maybe_gc(ctx); + } + dir_obj->dir = NULL; + } + + return result; +} + +// This function main purpose is to avoid warnings, such as: +// warning: comparison is always true due to limited range of data type [-Wtype-limits] +static inline term to_boxed_safe(uint64_t value, Context *ctx) +{ + if (value <= INT64_MAX) { + return term_make_maybe_boxed_int64(value, &ctx->heap); + } else { + return UNDEFINED_ATOM; + } +} + +static term nif_atomvm_posix_readdir(Context *ctx, int argc, term argv[]) +{ + UNUSED(argc); + + GlobalContext *glb = ctx->global; + + void *dir_obj_ptr; + if (UNLIKELY(!enif_get_resource( + erl_nif_env_from_context(ctx), argv[0], glb->posix_dir_resource_type, &dir_obj_ptr))) { + RAISE_ERROR(BADARG_ATOM); + } + struct PosixDir *dir_obj = (struct PosixDir *) dir_obj_ptr; + + errno = 0; + struct dirent *dir_result = readdir(dir_obj->dir); + if (dir_result == NULL) { + if (UNLIKELY(errno != 0)) { + return errno_to_error_tuple_maybe_gc(ctx); + } + + return globalcontext_make_atom(glb, ATOM_STR("\x3", "eof")); + } + + size_t name_len = strlen(dir_result->d_name); + if (UNLIKELY( + memory_ensure_free_opt(ctx, + BOXED_INT64_SIZE + term_binary_heap_size(name_len) + TUPLE_SIZE(3) + TUPLE_SIZE(2), + MEMORY_CAN_SHRINK) + != MEMORY_GC_OK)) { + RAISE_ERROR(OUT_OF_MEMORY_ATOM); + } + + term ino_no = to_boxed_safe(dir_result->d_ino, ctx); + + term name_term = term_create_uninitialized_binary(name_len, &ctx->heap, glb); + memcpy((void *) term_binary_data(name_term), dir_result->d_name, name_len); + + term dirent_atom = globalcontext_make_atom(glb, ATOM_STR("\x6", "dirent")); + + // {dirent, Inode, Name} + term dirent_term = term_alloc_tuple(3, &ctx->heap); + term_put_tuple_element(dirent_term, 0, dirent_atom); + term_put_tuple_element(dirent_term, 1, ino_no); + term_put_tuple_element(dirent_term, 2, name_term); + + // {ok, DirentTuple} + term result = term_alloc_tuple(2, &ctx->heap); + term_put_tuple_element(result, 0, OK_ATOM); + term_put_tuple_element(result, 1, dirent_term); + + return result; +} + +#endif + #if HAVE_OPEN && HAVE_CLOSE const struct Nif atomvm_posix_open_nif = { .base.type = NIFFunctionType, @@ -650,3 +821,17 @@ const struct Nif atomvm_posix_clock_settime_nif = { .nif_ptr = nif_atomvm_posix_clock_settime }; #endif +#if HAVE_OPENDIR && HAVE_READDIR && HAVE_CLOSEDIR +const struct Nif atomvm_posix_opendir_nif = { + .base.type = NIFFunctionType, + .nif_ptr = nif_atomvm_posix_opendir +}; +const struct Nif atomvm_posix_closedir_nif = { + .base.type = NIFFunctionType, + .nif_ptr = nif_atomvm_posix_closedir +}; +const struct Nif atomvm_posix_readdir_nif = { + .base.type = NIFFunctionType, + .nif_ptr = nif_atomvm_posix_readdir +}; +#endif diff --git a/src/libAtomVM/posix_nifs.h b/src/libAtomVM/posix_nifs.h index 061a155c8..4b425529e 100644 --- a/src/libAtomVM/posix_nifs.h +++ b/src/libAtomVM/posix_nifs.h @@ -53,6 +53,12 @@ extern const struct Nif atomvm_posix_unlink_nif; #if defined(HAVE_CLOCK_SETTIME) || defined(HAVE_SETTIMEOFDAY) extern const struct Nif atomvm_posix_clock_settime_nif; #endif +#if HAVE_OPENDIR && HAVE_READDIR && HAVE_CLOSEDIR +extern const ErlNifResourceTypeInit posix_dir_resource_type_init; +extern const struct Nif atomvm_posix_opendir_nif; +extern const struct Nif atomvm_posix_readdir_nif; +extern const struct Nif atomvm_posix_closedir_nif; +#endif /** * @brief Convenient function to return posix errors as atom. diff --git a/src/platforms/esp32/CMakeLists.txt b/src/platforms/esp32/CMakeLists.txt index 55407bc33..08fbfabd6 100644 --- a/src/platforms/esp32/CMakeLists.txt +++ b/src/platforms/esp32/CMakeLists.txt @@ -31,6 +31,12 @@ set(HAVE_MKFIFO "" CACHE INTERNAL "Have symbol mkfifo" FORCE) # in CMAKE_REQUIRED_INCLUDES as lwip includes freetos and many esp system components set(HAVE_SOCKET 1 CACHE INTERNAL "Have symbol socket" FORCE) +# opendir, closedir and readdir functions are not detected +# but they are available +set(HAVE_OPENDIR 1 CACHE INTERNAL "Have symbol opendir" FORCE) +set(HAVE_CLOSEDIR 1 CACHE INTERNAL "Have symbol closedir" FORCE) +set(HAVE_READDIR 1 CACHE INTERNAL "Have symbol readdir" FORCE) + list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/../../../CMakeModules") # Disable SMP with esp32 socs that have only one core diff --git a/tests/libs/eavmlib/CMakeLists.txt b/tests/libs/eavmlib/CMakeLists.txt index df2f30129..758764111 100644 --- a/tests/libs/eavmlib/CMakeLists.txt +++ b/tests/libs/eavmlib/CMakeLists.txt @@ -23,6 +23,7 @@ project(test_eavmlib) include(BuildErlang) set(ERLANG_MODULES + test_dir test_file test_ahttp_client test_port diff --git a/tests/libs/eavmlib/test_dir.erl b/tests/libs/eavmlib/test_dir.erl new file mode 100644 index 000000000..40e66126e --- /dev/null +++ b/tests/libs/eavmlib/test_dir.erl @@ -0,0 +1,38 @@ +% +% This file is part of AtomVM. +% +% Copyright 2024 Davide Bettio +% +% Licensed under the Apache License, Version 2.0 (the "License"); +% you may not use this file except in compliance with the License. +% You may obtain a copy of the License at +% +% http://www.apache.org/licenses/LICENSE-2.0 +% +% Unless required by applicable law or agreed to in writing, software +% distributed under the License is distributed on an "AS IS" BASIS, +% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +% See the License for the specific language governing permissions and +% limitations under the License. +% +% SPDX-License-Identifier: Apache-2.0 OR LGPL-2.1-or-later +% + +-module(test_dir). + +-export([test/0]). + +-include("etest.hrl"). + +test() -> + {ok, Dir} = atomvm:posix_opendir("."), + [eof | _Entries] = all_dir_entries(Dir, []), + ok = atomvm:posix_closedir(Dir). + +all_dir_entries(Dir, Acc) -> + case atomvm:posix_readdir(Dir) of + eof -> + [eof | Acc]; + {ok, {dirent, Inode, Name} = Dirent} when is_integer(Inode) and is_binary(Name) -> + all_dir_entries(Dir, [Dirent | Acc]) + end. diff --git a/tests/libs/eavmlib/tests.erl b/tests/libs/eavmlib/tests.erl index 914cd8bde..c8ceac567 100644 --- a/tests/libs/eavmlib/tests.erl +++ b/tests/libs/eavmlib/tests.erl @@ -24,6 +24,7 @@ start() -> etest:test([ + test_dir, test_file, test_port, test_timer_manager,