[FL-3386] Fast FAP Loader (#2790)

* FBT: build and add FastFAP(tm) sections
* Elf file: fast loading fap files. Really fast, like x15 times faster.
* fastfap.py: cleanup unused imports
* Toolchain: 23 version
* Elf File: remove log messages
* Scripts: fix file permissions
* FBT: explicit interpreter for fastfap invocation

Co-authored-by: あく <[email protected]>

firmware/targets/f18/api_symbols.csv

@@ -679,7 +679,8 @@ Function,+,elements_slightly_rounded_box,void,"Canvas*, uint8_t, uint8_t, uint8_
 Function,+,elements_slightly_rounded_frame,void,"Canvas*, uint8_t, uint8_t, uint8_t, uint8_t"
 Function,+,elements_string_fit_width,void,"Canvas*, FuriString*, uint8_t"
 Function,+,elements_text_box,void,"Canvas*, uint8_t, uint8_t, uint8_t, uint8_t, Align, Align, const char*, _Bool"
-Function,+,elf_resolve_from_hashtable,_Bool,"const ElfApiInterface*, const char*, Elf32_Addr*"
+Function,+,elf_resolve_from_hashtable,_Bool,"const ElfApiInterface*, uint32_t, Elf32_Addr*"
+Function,+,elf_symbolname_hash,uint32_t,const char*
 Function,+,empty_screen_alloc,EmptyScreen*,
 Function,+,empty_screen_free,void,EmptyScreen*
 Function,+,empty_screen_get_view,View*,EmptyScreen*

firmware/targets/f7/api_symbols.csv

@@ -808,7 +808,8 @@ Function,+,elements_slightly_rounded_box,void,"Canvas*, uint8_t, uint8_t, uint8_
 Function,+,elements_slightly_rounded_frame,void,"Canvas*, uint8_t, uint8_t, uint8_t, uint8_t"
 Function,+,elements_string_fit_width,void,"Canvas*, FuriString*, uint8_t"
 Function,+,elements_text_box,void,"Canvas*, uint8_t, uint8_t, uint8_t, uint8_t, Align, Align, const char*, _Bool"
-Function,+,elf_resolve_from_hashtable,_Bool,"const ElfApiInterface*, const char*, Elf32_Addr*"
+Function,+,elf_resolve_from_hashtable,_Bool,"const ElfApiInterface*, uint32_t, Elf32_Addr*"
+Function,+,elf_symbolname_hash,uint32_t,const char*
 Function,+,empty_screen_alloc,EmptyScreen*,
 Function,+,empty_screen_free,void,EmptyScreen*
 Function,+,empty_screen_get_view,View*,EmptyScreen*

lib/flipper_application/api_hashtable/api_hashtable.cpp

@@ -7,27 +7,22 @@
 
 bool elf_resolve_from_hashtable(
     const ElfApiInterface* interface,
-    const char* name,
+    uint32_t hash,
     Elf32_Addr* address) {
+    bool result = false;
     const HashtableApiInterface* hashtable_interface =
         static_cast<const HashtableApiInterface*>(interface);
-    bool result = false;
-    uint32_t gnu_sym_hash = elf_gnu_hash(name);
 
     sym_entry key = {
-        .hash = gnu_sym_hash,
+        .hash = hash,
         .address = 0,
     };
 
     auto find_res =
         std::lower_bound(hashtable_interface->table_cbegin, hashtable_interface->table_cend, key);
-    if((find_res == hashtable_interface->table_cend || (find_res->hash != gnu_sym_hash))) {
+    if((find_res == hashtable_interface->table_cend || (find_res->hash != hash))) {
         FURI_LOG_W(
-            TAG,
-            "Can't find symbol '%s' (hash %lx) @ %p!",
-            name,
-            gnu_sym_hash,
-            hashtable_interface->table_cbegin);
+            TAG, "Can't find symbol with hash %lx @ %p!", hash, hashtable_interface->table_cbegin);
         result = false;
     } else {
         result = true;
@@ -36,3 +31,7 @@ bool elf_resolve_from_hashtable(
 
     return result;
 }
+
+uint32_t elf_symbolname_hash(const char* s) {
+    return elf_gnu_hash(s);
+}

lib/flipper_application/api_hashtable/api_hashtable.h

@@ -19,15 +19,17 @@ struct sym_entry {
 /**
  * @brief Resolver for API entries using a pre-sorted table with hashes
  * @param interface pointer to HashtableApiInterface
- * @param name function name
+ * @param hash gnu hash of function name
  * @param address output for function address
  * @return true if the table contains a function
  */
 bool elf_resolve_from_hashtable(
     const ElfApiInterface* interface,
-    const char* name,
+    uint32_t hash,
     Elf32_Addr* address);
 
+uint32_t elf_symbolname_hash(const char* s);
+
 #ifdef __cplusplus
 }
 
@@ -48,8 +50,10 @@ struct HashtableApiInterface : public ElfApiInterface {
         .hash = elf_gnu_hash(#x), .address = (uint32_t)(static_cast<ret_type(*) args_type>(x)) \
     }
 
-#define API_VARIABLE(x, var_type) \
-    sym_entry { .hash = elf_gnu_hash(#x), .address = (uint32_t)(&(x)), }
+#define API_VARIABLE(x, var_type)                              \
+    sym_entry {                                                \
+        .hash = elf_gnu_hash(#x), .address = (uint32_t)(&(x)), \
+    }
 
 constexpr bool operator<(const sym_entry& k1, const sym_entry& k2) {
     return k1.hash < k2.hash;

lib/flipper_application/elf/elf_api_interface.h

@@ -11,6 +11,6 @@ typedef struct ElfApiInterface {
     uint16_t api_version_minor;
     bool (*resolver_callback)(
         const struct ElfApiInterface* interface,
-        const char* name,
+        uint32_t hash,
         Elf32_Addr* address);
 } ElfApiInterface;

lib/flipper_application/elf/elf_file.c

@@ -2,13 +2,15 @@
 #include "elf_file.h"
 #include "elf_file_i.h"
 #include "elf_api_interface.h"
+#include "../api_hashtable/api_hashtable.h"
 
 #define TAG "elf"
 
 #define ELF_NAME_BUFFER_LEN 32
 #define SECTION_OFFSET(e, n) ((e)->section_table + (n) * sizeof(Elf32_Shdr))
 #define IS_FLAGS_SET(v, m) (((v) & (m)) == (m))
 #define RESOLVER_THREAD_YIELD_STEP 30
+#define FAST_RELOCATION_VERSION 1
 
 // #define ELF_DEBUG_LOG 1
 
@@ -71,6 +73,7 @@ static ELFSection* elf_file_get_or_put_section(ELFFile* elf, const char* name) {
                 .size = 0,
                 .rel_count = 0,
                 .rel_offset = 0,
+                .fast_rel = NULL,
             });
         section_p = elf_file_get_section(elf, name);
     }
@@ -168,7 +171,8 @@ static ELFSection* elf_section_of(ELFFile* elf, int index) {
 static Elf32_Addr elf_address_of(ELFFile* elf, Elf32_Sym* sym, const char* sName) {
     if(sym->st_shndx == SHN_UNDEF) {
         Elf32_Addr addr = 0;
-        if(elf->api_interface->resolver_callback(elf->api_interface, sName, &addr)) {
+        uint32_t hash = elf_symbolname_hash(sName);
+        if(elf->api_interface->resolver_callback(elf->api_interface, hash, &addr)) {
             return addr;
         }
     } else {
@@ -424,6 +428,7 @@ typedef enum {
     SectionTypeSymTab = 1 << 3,
     SectionTypeStrTab = 1 << 4,
     SectionTypeDebugLink = 1 << 5,
+    SectionTypeFastRelData = 1 << 6,
 
     SectionTypeValid = SectionTypeSymTab | SectionTypeStrTab,
 } SectionType;
@@ -505,7 +510,8 @@ static SectionType elf_preload_section(
     // TODO: how to do it not by name?
     // .ARM: type 0x70000001, flags SHF_ALLOC | SHF_LINK_ORDER
     // .rel.ARM: type 0x9, flags SHT_REL
-    if(str_prefix(name, ".ARM.") || str_prefix(name, ".rel.ARM.")) {
+    if(str_prefix(name, ".ARM.") || str_prefix(name, ".rel.ARM.") ||
+       str_prefix(name, ".fast.rel.ARM.")) {
         FURI_LOG_D(TAG, "Ignoring ARM section");
         return SectionTypeUnused;
     }
@@ -536,11 +542,31 @@ static SectionType elf_preload_section(
 
     // Load link info section
     if(section_header->sh_flags & SHF_INFO_LINK) {
-        name = name + strlen(".rel");
+        if(str_prefix(name, ".rel")) {
+            name = name + strlen(".rel");
+            ELFSection* section_p = elf_file_get_or_put_section(elf, name);
+            section_p->rel_count = section_header->sh_size / sizeof(Elf32_Rel);
+            section_p->rel_offset = section_header->sh_offset;
+            return SectionTypeRelData;
+        } else {
+            FURI_LOG_E(TAG, "Unknown link info section '%s'", name);
+            return SectionTypeERROR;
+        }
+    }
+
+    // Load fast rel section
+    if(str_prefix(name, ".fast.rel")) {
+        name = name + strlen(".fast.rel");
         ELFSection* section_p = elf_file_get_or_put_section(elf, name);
-        section_p->rel_count = section_header->sh_size / sizeof(Elf32_Rel);
-        section_p->rel_offset = section_header->sh_offset;
-        return SectionTypeRelData;
+        section_p->fast_rel = malloc(sizeof(ELFSection));
+
+        if(!elf_load_section_data(elf, section_p->fast_rel, section_header)) {
+            FURI_LOG_E(TAG, "Error loading section '%s'", name);
+            return SectionTypeERROR;
+        }
+
+        FURI_LOG_D(TAG, "Loaded fast rel section for '%s'", name);
+        return SectionTypeFastRelData;
     }
 
     // Load symbol table
@@ -571,8 +597,90 @@ static SectionType elf_preload_section(
     return SectionTypeUnused;
 }
 
+static Elf32_Addr elf_address_of_by_hash(ELFFile* elf, uint32_t hash) {
+    Elf32_Addr addr = 0;
+    if(elf->api_interface->resolver_callback(elf->api_interface, hash, &addr)) {
+        return addr;
+    }
+    return ELF_INVALID_ADDRESS;
+}
+
+static bool elf_relocate_fast(ELFFile* elf, ELFSection* s) {
+    UNUSED(elf);
+    const uint8_t* start = s->fast_rel->data;
+    const uint8_t version = *start;
+
+    if(version != FAST_RELOCATION_VERSION) {
+        FURI_LOG_E(TAG, "Unsupported fast relocation version %d", version);
+        return false;
+    }
+    start += 1;
+
+    const uint32_t records_count = *((uint32_t*)start);
+    start += 4;
+    FURI_LOG_D(TAG, "Fast relocation records count: %ld", records_count);
+
+    for(uint32_t i = 0; i < records_count; i++) {
+        bool is_section = (*start & (0x1 << 7)) ? true : false;
+        uint8_t type = *start & 0x7F;
+        start += 1;
+        uint32_t hash_or_section_index = *((uint32_t*)start);
+        start += 4;
+
+        uint32_t section_value = ELF_INVALID_ADDRESS;
+        if(is_section) {
+            section_value = *((uint32_t*)start);
+            start += 4;
+        }
+
+        const uint32_t offsets_count = *((uint32_t*)start);
+        start += 4;
+
+        FURI_LOG_D(
+            TAG,
+            "Fast relocation record %ld: is_section=%d, type=%d, hash_or_section_index=%lX, offsets_count=%ld",
+            i,
+            is_section,
+            type,
+            hash_or_section_index,
+            offsets_count);
+
+        Elf32_Addr address = 0;
+        if(is_section) {
+            ELFSection* symSec = elf_section_of(elf, hash_or_section_index);
+            if(symSec) {
+                address = ((Elf32_Addr)symSec->data) + section_value;
+            }
+        } else {
+            address = elf_address_of_by_hash(elf, hash_or_section_index);
+        }
+
+        if(address == ELF_INVALID_ADDRESS) {
+            FURI_LOG_E(TAG, "Failed to resolve address for hash %lX", hash_or_section_index);
+            return false;
+        }
+
+        for(uint32_t j = 0; j < offsets_count; j++) {
+            uint32_t offset = *((uint32_t*)start) & 0x00FFFFFF;
+            start += 3;
+            // FURI_LOG_I(TAG, "  Fast relocation offset %ld: %ld", j, offset);
+            Elf32_Addr relAddr = ((Elf32_Addr)s->data) + offset;
+            elf_relocate_symbol(elf, relAddr, type, address);
+        }
+    }
+
+    aligned_free(s->fast_rel->data);
+    free(s->fast_rel);
+    s->fast_rel = NULL;
+
+    return true;
+}
+
 static bool elf_relocate_section(ELFFile* elf, ELFSection* section) {
-    if(section->rel_count) {
+    if(section->fast_rel) {
+        FURI_LOG_D(TAG, "Fast relocating section");
+        return elf_relocate_fast(elf, section);
+    } else if(section->rel_count) {
         FURI_LOG_D(TAG, "Relocating section");
         return elf_relocate(elf, section);
     } else {
@@ -630,6 +738,10 @@ void elf_file_free(ELFFile* elf) {
             if(itref->value.data) {
                 aligned_free(itref->value.data);
             }
+            if(itref->value.fast_rel) {
+                aligned_free(itref->value.fast_rel->data);
+                free(itref->value.fast_rel);
+            }
             free((void*)itref->key);
         }
 

lib/flipper_application/elf/elf_file_i.h

@@ -13,14 +13,18 @@ DICT_DEF2(AddressCache, int, M_DEFAULT_OPLIST, Elf32_Addr, M_DEFAULT_OPLIST)
  */
 typedef int32_t(entry_t)(void*);
 
-typedef struct {
+typedef struct ELFSection ELFSection;
+
+struct ELFSection {
     void* data;
-    uint16_t sec_idx;
     Elf32_Word size;
 
     size_t rel_count;
     Elf32_Off rel_offset;
-} ELFSection;
+    ELFSection* fast_rel;
+
+    uint16_t sec_idx;
+};
 
 DICT_DEF2(ELFSectionDict, const char*, M_CSTR_OPLIST, ELFSection, M_POD_OPLIST)
 

lib/flipper_application/plugins/composite_resolver.c

@@ -13,12 +13,12 @@ struct CompositeApiResolver {
 
 static bool composite_api_resolver_callback(
     const ElfApiInterface* interface,
-    const char* name,
+    uint32_t hash,
     Elf32_Addr* address) {
     CompositeApiResolver* resolver = (CompositeApiResolver*)interface;
     for
         M_EACH(interface, resolver->interfaces, ElfApiInterfaceList_t) {
-            if((*interface)->resolver_callback(*interface, name, address)) {
+            if((*interface)->resolver_callback(*interface, hash, address)) {
                 return true;
             }
         }