From fd62240a8ee072f81ca9324cc895269fd62152a5 Mon Sep 17 00:00:00 2001 From: Paul Guyot Date: Mon, 21 Aug 2023 21:30:53 +0200 Subject: [PATCH] Use a pointer for PC instead of an instruction counter Also mark code in modules as const. This optimization brings an additional 20% speed upgrade to the Sudoku benchmark. Signed-off-by: Paul Guyot --- src/libAtomVM/module.c | 4 +- src/libAtomVM/module.h | 2 +- src/libAtomVM/opcodesswitch.h | 2192 ++++++++++++--------------------- src/libAtomVM/stacktrace.c | 8 +- 4 files changed, 822 insertions(+), 1384 deletions(-) diff --git a/src/libAtomVM/module.c b/src/libAtomVM/module.c index 579a9167b..1fb2d52e3 100644 --- a/src/libAtomVM/module.c +++ b/src/libAtomVM/module.c @@ -60,7 +60,7 @@ static void *module_uncompress_literals(const uint8_t *litT, int size); #endif static struct LiteralEntry *module_build_literals_table(const void *literalsBuf); -static void module_add_label(Module *mod, int index, void *ptr); +static void module_add_label(Module *mod, int index, const uint8_t *ptr); static enum ModuleLoadResult module_build_imported_functions_table(Module *this_module, uint8_t *table_data, GlobalContext *glb); static void parse_line_table(uint16_t **line_refs, struct ModuleFilename **filenames, uint8_t *data, size_t len); @@ -235,7 +235,7 @@ term module_get_exported_functions(Module *this_module, Heap *heap, GlobalContex return result_list; } -static void module_add_label(Module *mod, int index, void *ptr) +static void module_add_label(Module *mod, int index, const uint8_t *ptr) { mod->labels[index] = ptr; } diff --git a/src/libAtomVM/module.h b/src/libAtomVM/module.h index 16266306a..d4b838a67 100644 --- a/src/libAtomVM/module.h +++ b/src/libAtomVM/module.h @@ -114,7 +114,7 @@ struct Module const struct ExportedFunction **imported_funcs; - void **labels; + const uint8_t **labels; void *literals_data; diff --git a/src/libAtomVM/opcodesswitch.h b/src/libAtomVM/opcodesswitch.h index 82e36c2ef..6c3739597 100644 --- a/src/libAtomVM/opcodesswitch.h +++ b/src/libAtomVM/opcodesswitch.h @@ -81,10 +81,10 @@ #define COMPACT_NBITS_VALUE 0x18 #ifdef IMPL_EXECUTE_LOOP -#define SET_ERROR(error_type_atom) \ - x_regs[0] = ERROR_ATOM; \ - x_regs[1] = error_type_atom; \ - x_regs[2] = stacktrace_create_raw(ctx, mod, i, ERROR_ATOM); \ +#define SET_ERROR(error_type_atom) \ + x_regs[0] = ERROR_ATOM; \ + x_regs[1] = error_type_atom; \ + x_regs[2] = stacktrace_create_raw(ctx, mod, pc - code, ERROR_ATOM); // Override nifs.h RAISE_ERROR macro #ifdef RAISE_ERROR @@ -121,9 +121,9 @@ typedef dreg_t dreg_unsafe_t; #endif -#define DECODE_COMPACT_TERM(dest_term, code_chunk, base_index, off) \ +#define DECODE_COMPACT_TERM(dest_term, decode_pc) \ { \ - uint8_t first_byte = (code_chunk[(base_index) + (off)]); \ + uint8_t first_byte = *(decode_pc)++; \ switch (first_byte & 0xF) { \ case COMPACT_LARGE_LITERAL: \ case COMPACT_LITERAL: \ @@ -131,12 +131,10 @@ typedef dreg_t dreg_unsafe_t; case 0: \ case 2: \ dest_term = term_from_int4(first_byte >> 4); \ - off += 1; \ break; \ \ case 1: \ - dest_term = term_from_int(((first_byte & 0xE0) << 3) | code_chunk[(base_index) + (off) + 1]); \ - off += 2; \ + dest_term = term_from_int(((first_byte & 0xE0) << 3) | *(decode_pc)++); \ break; \ \ default: \ @@ -150,11 +148,10 @@ typedef dreg_t dreg_unsafe_t; switch (((first_byte) >> 3) & 0x3) { \ case 0: \ case 2: \ - off += 1; \ break; \ \ default: \ - fprintf(stderr, "Operand not a small integer: %x, or unsupported encoding\n", (first_byte)); \ + fprintf(stderr, "Operand not a small integer: %x, or unsupported encoding\n", (first_byte)); \ AVM_ABORT(); \ break; \ } \ @@ -163,41 +160,37 @@ typedef dreg_t dreg_unsafe_t; case COMPACT_ATOM: \ case COMPACT_XREG: \ case COMPACT_YREG: \ - off += 1; \ break; \ \ case COMPACT_EXTENDED: \ switch (first_byte) { \ case COMPACT_EXTENDED_LITERAL: { \ - uint8_t ext = (code_chunk[(base_index) + (off) + 1] & 0xF); \ - if (ext == 0) { \ - off += 2; \ - } else if (ext == 0x8) { \ - off += 3; \ - } else { \ + uint8_t ext = ((*(decode_pc)++) & 0xF); \ + if (ext == 0x8) { \ + (decode_pc)++; \ + } else if (ext != 0) { \ AVM_ABORT(); \ } \ break; \ } \ case COMPACT_EXTENDED_ALLOCATION_LIST: { \ - uint8_t len = (code_chunk[(base_index) + (off) + 1] >> 4); \ - off += (len * 2); \ + uint8_t len = (*(decode_pc)++) >> 4; \ + (decode_pc) += (--len * 2); \ break; \ } \ case COMPACT_EXTENDED_TYPED_REGISTER: { \ - uint8_t reg_byte = code_chunk[(base_index) + (off) + 1]; \ + uint8_t reg_byte = *(decode_pc)++; \ if (((reg_byte & 0x0F) != COMPACT_XREG) \ && ((reg_byte & 0x0F) != COMPACT_YREG)) { \ - fprintf(stderr, "Unexpected reg byte %x @ %" PRIuPTR "\n", (int) reg_byte, (base_index) + (off) + 1); \ + fprintf(stderr, "Unexpected reg byte %x @ %" PRIuPTR "\n", (int) reg_byte, (uintptr_t) ((decode_pc) - 1)); \ AVM_ABORT(); \ } \ - off += 2; \ int type_index; \ - DECODE_LITERAL(type_index, code_chunk, base_index, off) \ + DECODE_LITERAL(type_index, decode_pc) \ break; \ } \ default: \ - fprintf(stderr, "Unexpected extended %x @ %" PRIuPTR "\n", (int) first_byte, (base_index) + (off) + 1); \ + fprintf(stderr, "Unexpected extended %x @ %" PRIuPTR "\n", (int) first_byte, (uintptr_t) ((decode_pc) - 1)); \ AVM_ABORT(); \ break; \ } \ @@ -207,12 +200,12 @@ typedef dreg_t dreg_unsafe_t; case COMPACT_LARGE_ATOM: \ switch (first_byte & COMPACT_LARGE_IMM_MASK) { \ case COMPACT_11BITS_VALUE: \ - off += 2; \ + (decode_pc)++; \ break; \ \ case COMPACT_NBITS_VALUE: \ /* TODO: when first_byte >> 5 is 7, a different encoding is used */ \ - off += (first_byte >> 5) + 3; \ + (decode_pc) += (first_byte >> 5) + 2; \ break; \ \ default: \ @@ -222,7 +215,7 @@ typedef dreg_t dreg_unsafe_t; break; \ \ case COMPACT_LARGE_YREG: \ - off += 2; \ + (decode_pc)++; \ break; \ \ default: \ @@ -232,92 +225,85 @@ typedef dreg_t dreg_unsafe_t; } \ } -#define DECODE_EXTENDED_LIST_TAG(code_chunk, base_index, off) \ +#define DECODE_EXTENDED_LIST_TAG(decode_pc) \ { \ - if ((code_chunk[(base_index) + (off)]) != COMPACT_EXTENDED_LIST) { \ - fprintf(stderr, "Unexpected operand, expected a list, got %x\n", code_chunk[(base_index) + (off)]); \ + if ((*(decode_pc)++) != COMPACT_EXTENDED_LIST) { \ + fprintf(stderr, "Unexpected operand, expected a list, got %x\n", (decode_pc)[-1]);\ AVM_ABORT(); \ } \ - off++; \ } #ifdef ENABLE_TRACE -#define DECODE_DEST_REGISTER(dreg, code_chunk, base_index, off) \ +#define DECODE_DEST_REGISTER(dreg, decode_pc) \ { \ - uint8_t first_byte = code_chunk[(base_index) + (off)]; \ + uint8_t first_byte = *(decode_pc)++; \ uint8_t reg_type = first_byte & 0xF; \ (dreg).reg_type = reg_type; \ switch (reg_type) { \ case COMPACT_XREG: \ case COMPACT_YREG: \ - (dreg).index = code_chunk[(base_index) + (off)] >> 4; \ - off += 1; \ - break; \ + (dreg).index = first_byte >> 4; \ +- break; \ case COMPACT_LARGE_YREG: \ - (dreg).index = (((first_byte & 0xE0) << 3) | code_chunk[(base_index) + (off) + 1]); \ - off += 2; \ + (dreg).index = (((first_byte & 0xE0) << 3) | *(decode_pc)++); \ break; \ default: \ AVM_ABORT(); \ } \ } -#define DECODE_DEST_REGISTER_UNSAFE(dreg, code_chunk, base_index, off) \ - DECODE_DEST_REGISTER(dreg, code_chunk, base_index, off) +#define DECODE_DEST_REGISTER_UNSAFE(dreg, decode_pc) \ + DECODE_DEST_REGISTER(dreg, decode_pc) #else -#define DECODE_DEST_REGISTER(dreg, code_chunk, base_index, off) \ +#define DECODE_DEST_REGISTER(dreg, decode_pc) \ { \ UNUSED(dreg); \ - uint8_t first_byte = code_chunk[(base_index) + (off)]; \ + uint8_t first_byte = *(decode_pc)++; \ uint8_t reg_type = first_byte & 0xF; \ switch (reg_type) { \ case COMPACT_XREG: \ case COMPACT_YREG: \ - off += 1; \ break; \ case COMPACT_LARGE_YREG: \ - off += 2; \ + (decode_pc)++; \ break; \ default: \ AVM_ABORT(); \ } \ } -#define DECODE_DEST_REGISTER_UNSAFE(dreg, code_chunk, base_index, off) \ - DECODE_DEST_REGISTER(dreg, code_chunk, base_index, off) +#define DECODE_DEST_REGISTER_UNSAFE(dreg, decode_pc) \ + DECODE_DEST_REGISTER(dreg, decode_pc) #endif -#define DECODE_FP_REGISTER(freg, code_chunk, base_index, off) \ +#define DECODE_FP_REGISTER(freg, decode_pc) \ { \ - if ((code_chunk[(base_index) + (off)]) != COMPACT_EXTENDED_FP_REGISTER) { \ - fprintf(stderr, "Unexpected operand, expected an fp register, got %x\n", code_chunk[(base_index) + (off)]); \ + if ((*(decode_pc)++) != COMPACT_EXTENDED_FP_REGISTER) { \ + fprintf(stderr, "Unexpected operand, expected an fp register, got %x\n", (decode_pc)[-1]); \ AVM_ABORT(); \ } \ - off++; \ - DECODE_LITERAL(freg, code_chunk, base_index, off); \ + DECODE_LITERAL(freg, decode_pc); \ if (freg > MAX_REG) { \ fprintf(stderr, "FP register index %d > MAX_REG = %d\n", freg, MAX_REG); \ AVM_ABORT(); \ } \ } -#define DECODE_VALUE32(val, code_chunk, base_index, off) \ +#define DECODE_VALUE32(val, decode_pc) \ { \ - uint8_t first_byte = (code_chunk[(base_index) + (off)]); \ + uint8_t first_byte = *(decode_pc)++; \ switch (((first_byte) >> 3) & 0x3) { \ case 0: \ case 2: \ val = first_byte >> 4; \ - off += 1; \ break; \ \ case 1: \ - val = ((first_byte & 0xE0) << 3) | code_chunk[(base_index) + (off) + 1]; \ - off += 2; \ + val = ((first_byte & 0xE0) << 3) | *(decode_pc)++; \ break; \ \ case 3: { \ @@ -329,28 +315,25 @@ typedef dreg_t dreg_unsafe_t; val = 0; \ for (uint8_t vi = 0; vi < sz; vi++) { \ val <<= 8; \ - val |= code[(base_index) + (off) + 1 + vi]; \ + val |= *(decode_pc)++; \ } \ - off += 1 + sz; \ break; \ } \ default: UNREACHABLE(); /* help gcc 8.4 */ \ } \ } -#define DECODE_VALUE64(val, code_chunk, base_index, off) \ +#define DECODE_VALUE64(val, decode_pc) \ { \ - uint8_t first_byte = (code_chunk[(base_index) + (off)]); \ + uint8_t first_byte = *(decode_pc)++; \ switch (((first_byte) >> 3) & 0x3) { \ case 0: \ case 2: \ val = first_byte >> 4; \ - off += 1; \ break; \ \ case 1: \ - val = ((first_byte & 0xE0) << 3) | code_chunk[(base_index) + (off) + 1]; \ - off += 2; \ + val = ((first_byte & 0xE0) << 3) | *(decode_pc)++; \ break; \ \ case 3: { \ @@ -362,81 +345,80 @@ typedef dreg_t dreg_unsafe_t; val = 0; \ for (uint8_t vi = 0; vi < sz; vi++) { \ val <<= 8; \ - val |= code[(base_index) + (off) + 1 + vi]; \ + val |= *(decode_pc)++; \ } \ - off += 1 + sz; \ break; \ } \ } \ } -#define DECODE_ATOM(atom, code_chunk, base_index, off) \ +#define DECODE_ATOM(atom, decode_pc) \ { \ - if (UNLIKELY(((code_chunk[(base_index) + (off)]) & 0x7) != COMPACT_ATOM)) { \ - fprintf(stderr, "Unexpected operand, expected an atom (%x)\n", (code_chunk[(base_index) + (off)])); \ + if (UNLIKELY((*(decode_pc) & 0x7) != COMPACT_ATOM)) { \ + fprintf(stderr, "Unexpected operand, expected an atom (%x)\n", *(decode_pc)); \ AVM_ABORT(); \ } \ uint32_t atom_ix; \ - DECODE_VALUE32(atom_ix, code_chunk, base_index, off); \ + DECODE_VALUE32(atom_ix, decode_pc); \ atom = module_get_atom_term_by_id(mod, atom_ix); \ } -#define DECODE_LABEL(label, code_chunk, base_index, off) \ +#define DECODE_LABEL(label, decode_pc) \ { \ - if (UNLIKELY(((code_chunk[(base_index) + (off)]) & 0x7) != COMPACT_LABEL)) { \ - fprintf(stderr, "Unexpected operand, expected a label (%x)\n", (code_chunk[(base_index) + (off)])); \ + if (UNLIKELY((*(decode_pc) & 0x7) != COMPACT_LABEL)) { \ + fprintf(stderr, "Unexpected operand, expected a label (%x)\n", *(decode_pc)); \ AVM_ABORT(); \ } \ - DECODE_VALUE32(label, code_chunk, base_index, off); \ + DECODE_VALUE32(label, decode_pc); \ } -#define DECODE_LITERAL(literal, code_chunk, base_index, off) \ +#define DECODE_LITERAL(literal, decode_pc) \ { \ - if (UNLIKELY(((code_chunk[(base_index) + (off)]) & 0x7) != COMPACT_LITERAL)) { \ - fprintf(stderr, "Unexpected operand, expected a literal (%x)\n", (code_chunk[(base_index) + (off)])); \ + if (UNLIKELY((*(decode_pc) & 0x7) != COMPACT_LITERAL)) { \ + fprintf(stderr, "Unexpected operand, expected a literal (%x)\n", *(decode_pc)); \ AVM_ABORT(); \ } \ - DECODE_VALUE32(literal, code_chunk, base_index, off); \ + DECODE_VALUE32(literal, decode_pc); \ } -#define DECODE_INTEGER(integer, code_chunk, base_index, off) \ +#define DECODE_INTEGER(integer, decode_pc) \ { \ - if (UNLIKELY(((code_chunk[(base_index) + (off)]) & 0x7) != COMPACT_INTEGER)) { \ - fprintf(stderr, "Unexpected operand, expected an integer (%x)\n", (code_chunk[(base_index) + (off)])); \ + if (UNLIKELY((*(decode_pc) & 0x7) != COMPACT_INTEGER)) { \ + fprintf(stderr, "Unexpected operand, expected an integer (%x)\n", *(decode_pc)); \ AVM_ABORT(); \ } \ - DECODE_VALUE64(integer, code_chunk, base_index, off); \ + DECODE_VALUE64(integer, decode_pc); \ } -#define DECODE_XREG(reg, code_chunk, base_index, off) \ +#define DECODE_XREG(reg, decode_pc) \ { \ - if (UNLIKELY(((code_chunk[(base_index) + (off)]) & 0x7) != COMPACT_XREG)) { \ - fprintf(stderr, "Unexpected operand, expected an xreg (%x)\n", (code_chunk[(base_index) + (off)])); \ + if (UNLIKELY((*(decode_pc) & 0x7) != COMPACT_XREG)) { \ + fprintf(stderr, "Unexpected operand, expected an xreg (%x)\n", *(decode_pc)); \ AVM_ABORT(); \ } \ - DECODE_VALUE32(reg, code_chunk, base_index, off); \ + DECODE_VALUE32(reg, decode_pc); \ } -#define DECODE_YREG(reg, code_chunk, base_index, off) \ +#define DECODE_YREG(reg, decode_pc) \ { \ - if (UNLIKELY(((code_chunk[(base_index) + (off)]) & 0x7) != COMPACT_YREG)) { \ - fprintf(stderr, "Unexpected operand, expected a yreg (%x)\n", (code_chunk[(base_index) + (off)])); \ + if (UNLIKELY((*(decode_pc) & 0x7) != COMPACT_YREG)) { \ + fprintf(stderr, "Unexpected operand, expected a yreg (%x)\n", *(decode_pc)); \ AVM_ABORT(); \ } \ - DECODE_VALUE32(reg, code_chunk, base_index, off); \ + DECODE_VALUE32(reg, decode_pc); \ } -#define DECODE_ALLOCATOR_LIST(need, code_chunk, base_index, off) \ - if (IS_EXTENDED_ALLOCATOR(code_chunk, base_index, off)) { \ +#define DECODE_ALLOCATOR_LIST(need, decode_pc) \ + if (IS_EXTENDED_ALLOCATOR(decode_pc)) { \ need = 0; \ - off++; /* skip list tag */ \ + (decode_pc)++; /* skip list tag */ \ uint32_t list_size; \ - DECODE_LITERAL(list_size, code_chunk, base_index, off); \ + DECODE_LITERAL(list_size, (decode_pc)); \ uint32_t allocator_tag; \ uint32_t allocator_size; \ for (uint32_t j = 0; j < list_size; j++) { \ - DECODE_LITERAL(allocator_tag, code_chunk, base_index, off); \ - DECODE_LITERAL(allocator_size, code_chunk, base_index, off); \ + DECODE_LITERAL(allocator_tag, (decode_pc)); \ + DECODE_LITERAL(allocator_size, (decode_pc)); \ if (allocator_tag == COMPACT_EXTENDED_ALLOCATOR_LIST_TAG_FLOATS) { \ allocator_size *= FLOAT_SIZE; \ } else if (allocator_tag == COMPACT_EXTENDED_ALLOCATOR_LIST_TAG_FUNS) { \ @@ -445,7 +427,7 @@ typedef dreg_t dreg_unsafe_t; need += allocator_size; \ } \ } else { \ - DECODE_LITERAL(need, code_chunk, base_index, off); \ + DECODE_LITERAL(need, decode_pc); \ } #endif @@ -466,9 +448,9 @@ typedef struct #define T_DEST_REG_UNSAFE(dreg_unsafe) \ ((dreg).base == x_regs) ? 'x' : 'y', ((dreg).index) -#define DECODE_COMPACT_TERM(dest_term, code_chunk, base_index, off) \ +#define DECODE_COMPACT_TERM(dest_term, decode_pc) \ { \ - uint8_t first_byte = (code_chunk[(base_index) + (off)]); \ + uint8_t first_byte = *(decode_pc)++; \ switch (first_byte & 0xF) { \ case COMPACT_LARGE_LITERAL: \ case COMPACT_LITERAL: \ @@ -476,12 +458,10 @@ typedef struct case 0: \ case 2: \ dest_term = term_from_int4(first_byte >> 4); \ - off += 1; \ break; \ \ case 1: \ - dest_term = term_from_int(((first_byte & 0xE0) << 3) | code_chunk[(base_index) + (off) + 1]); \ - off += 2; \ + dest_term = term_from_int(((first_byte & 0xE0) << 3) | *(decode_pc)++); \ break; \ \ default: \ @@ -493,7 +473,6 @@ typedef struct \ case COMPACT_INTEGER: \ dest_term = term_from_int4(first_byte >> 4); \ - off += 1; \ break; \ \ case COMPACT_ATOM: \ @@ -502,31 +481,26 @@ typedef struct } else { \ dest_term = module_get_atom_term_by_id(mod, first_byte >> 4); \ } \ - off += 1; \ break; \ \ case COMPACT_XREG: \ dest_term = x_regs[first_byte >> 4]; \ - off += 1; \ break; \ \ case COMPACT_YREG: \ dest_term = ctx->e[first_byte >> 4]; \ - off += 1; \ break; \ \ case COMPACT_EXTENDED: \ switch (first_byte) { \ case COMPACT_EXTENDED_LITERAL: { \ - uint8_t first_extended_byte = code_chunk[(base_index) + (off) + 1]; \ + uint8_t first_extended_byte = *(decode_pc)++; \ if (!(first_extended_byte & 0xF)) { \ dest_term = module_load_literal(mod, first_extended_byte >> 4, ctx); \ - off += 2; \ } else if ((first_extended_byte & 0xF) == 0x8) { \ - uint8_t byte_1 = code_chunk[(base_index) + (off) + 2]; \ + uint8_t byte_1 = *(decode_pc)++; \ uint16_t index = (((uint16_t) first_extended_byte & 0xE0) << 3) | byte_1; \ dest_term = module_load_literal(mod, index, ctx); \ - off += 3; \ } else { \ VM_ABORT(); \ } \ @@ -537,15 +511,14 @@ typedef struct break; \ } \ case COMPACT_EXTENDED_TYPED_REGISTER: { \ - uint8_t reg_byte = code_chunk[(base_index) + (off) + 1]; \ + uint8_t reg_byte = *(decode_pc)++; \ if ((reg_byte & 0x0F) == COMPACT_XREG) { \ dest_term = x_regs[reg_byte >> 4]; \ } else { \ dest_term = ctx->e[reg_byte >> 4]; \ } \ - off += 2; \ int type_index; \ - DECODE_LITERAL(type_index, code_chunk, base_index, off) \ + DECODE_LITERAL(type_index, decode_pc) \ break; \ } \ default: \ @@ -557,8 +530,7 @@ typedef struct case COMPACT_LARGE_ATOM: \ switch (first_byte & COMPACT_LARGE_IMM_MASK) { \ case COMPACT_11BITS_VALUE: \ - dest_term = module_get_atom_term_by_id(mod, ((first_byte & 0xE0) << 3) | code_chunk[(base_index) + (off) + 1]); \ - off += 2; \ + dest_term = module_get_atom_term_by_id(mod, ((first_byte & 0xE0) << 3) | *(decode_pc)++); \ break; \ \ default: \ @@ -570,17 +542,18 @@ typedef struct case COMPACT_LARGE_INTEGER: \ switch (first_byte & COMPACT_LARGE_IMM_MASK) { \ case COMPACT_11BITS_VALUE: \ - dest_term = term_from_int11(((first_byte & 0xE0) << 3) | code_chunk[(base_index) + (off) + 1]); \ - off += 2; \ + dest_term = term_from_int11(((first_byte & 0xE0) << 3) | *(decode_pc)++); \ break; \ \ - case COMPACT_NBITS_VALUE: \ - dest_term = large_integer_to_term(ctx, (code_chunk) + (base_index) + (off), &(off)); \ + case COMPACT_NBITS_VALUE: { \ + size_t num_bytes = (first_byte >> 5) + 2; \ + dest_term = large_integer_to_term(ctx, num_bytes, decode_pc); \ if (UNLIKELY(term_is_invalid_term(dest_term))) { \ HANDLE_ERROR(); \ } \ + (decode_pc) += num_bytes; \ break; \ - \ + } \ default: \ VM_ABORT(); \ break; \ @@ -589,8 +562,7 @@ typedef struct \ case COMPACT_LARGE_YREG: \ if (LIKELY((first_byte & COMPACT_LARGE_IMM_MASK) == COMPACT_11BITS_VALUE)) { \ - dest_term = ctx->e[((first_byte & 0xE0) << 3) | code_chunk[(base_index) + (off) + 1]]; \ - off += 2; \ + dest_term = ctx->e[((first_byte & 0xE0) << 3) | *(decode_pc)++]; \ } else { \ VM_ABORT(); \ } \ @@ -621,29 +593,26 @@ typedef struct } \ } -#define DECODE_EXTENDED_LIST_TAG(code_chunk, base_index, off) \ +#define DECODE_EXTENDED_LIST_TAG(decode_pc) \ { \ - off++; \ + decode_pc++; \ } -#define DECODE_DEST_REGISTER(dreg, code_chunk, base_index, off) \ +#define DECODE_DEST_REGISTER(dreg, decode_pc) \ { \ - uint8_t first_byte = code_chunk[(base_index) + (off)]; \ + uint8_t first_byte = *(decode_pc)++; \ uint8_t reg_type = first_byte & 0xF; \ uint8_t reg_index = (first_byte >> 4); \ switch (reg_type) { \ case COMPACT_XREG: \ (dreg) = x_regs + reg_index; \ - off++; \ break; \ case COMPACT_YREG: \ (dreg) = ctx->e + reg_index; \ - off++; \ break; \ case COMPACT_LARGE_YREG: \ if (LIKELY((first_byte & COMPACT_LARGE_IMM_MASK) == COMPACT_11BITS_VALUE)) { \ - (dreg) = ctx->e + (((first_byte & 0xE0) << 3) | code_chunk[(base_index) + (off) + 1]); \ - off += 2; \ + (dreg) = ctx->e + (((first_byte & 0xE0) << 3) | *(decode_pc)++); \ } else { \ VM_ABORT(); \ } \ @@ -653,27 +622,24 @@ typedef struct } \ } -#define DECODE_DEST_REGISTER_UNSAFE(dreg_unsafe, code_chunk, base_index, off) \ +#define DECODE_DEST_REGISTER_UNSAFE(dreg_unsafe, decode_pc) \ { \ - uint8_t first_byte = code_chunk[(base_index) + (off)]; \ + uint8_t first_byte = *(decode_pc)++; \ uint8_t reg_type = first_byte & 0xF; \ uint8_t reg_index = (first_byte >> 4); \ switch (reg_type) { \ case COMPACT_XREG: \ (dreg_unsafe).base = x_regs; \ (dreg_unsafe).index = reg_index; \ - off++; \ break; \ case COMPACT_YREG: \ (dreg_unsafe).base = ctx->e; \ (dreg_unsafe).index = reg_index; \ - off++; \ break; \ case COMPACT_LARGE_YREG: \ if (LIKELY((first_byte & COMPACT_LARGE_IMM_MASK) == COMPACT_11BITS_VALUE)) { \ (dreg_unsafe).base = ctx->e; \ - (dreg_unsafe).index = (((first_byte & 0xE0) << 3) | code_chunk[(base_index) + (off) + 1]); \ - off += 2; \ + (dreg_unsafe).index = (((first_byte & 0xE0) << 3) | *(decode_pc)++); \ } else { \ VM_ABORT(); \ } \ @@ -683,25 +649,23 @@ typedef struct } \ } -#define DECODE_FP_REGISTER(freg, code_chunk, base_index, off) \ +#define DECODE_FP_REGISTER(freg, decode_pc) \ { \ - off++; \ - DECODE_LITERAL(freg, code_chunk, base_index, off); \ + (decode_pc)++; \ + DECODE_LITERAL(freg, decode_pc); \ } -#define DECODE_VALUE(val, code_chunk, base_index, off) \ +#define DECODE_VALUE(val, decode_pc) \ { \ - uint8_t first_byte = (code_chunk[(base_index) + (off)]); \ + uint8_t first_byte = *(decode_pc)++; \ switch (((first_byte) >> 3) & 0x3) { \ case 0: \ case 2: \ val = first_byte >> 4; \ - off += 1; \ break; \ \ case 1: \ - val = ((first_byte & 0xE0) << 3) | code_chunk[(base_index) + (off) + 1]; \ - off += 2; \ + val = ((first_byte & 0xE0) << 3) | *(decode_pc)++; \ break; \ \ case 3: { \ @@ -709,48 +673,47 @@ typedef struct val = 0; \ for (uint8_t vi = 0; vi < sz; vi++) { \ val <<= 8; \ - val |= code_chunk[(base_index) + (off) + 1 + vi]; \ + val |= *(decode_pc)++; \ } \ - off += 1 + sz; \ break; \ } \ default: UNREACHABLE(); /* help gcc 8.4 */ \ } \ } -#define DECODE_ATOM(atom, code_chunk, base_index, off) \ +#define DECODE_ATOM(atom, decode_pc) \ { \ uint32_t atom_ix; \ - DECODE_VALUE(atom_ix, code_chunk, base_index, off); \ + DECODE_VALUE(atom_ix, decode_pc); \ atom = module_get_atom_term_by_id(mod, atom_ix); \ } -#define DECODE_LABEL(label, code_chunk, base_index, off) \ - DECODE_VALUE(label, code_chunk, base_index, off) +#define DECODE_LABEL(label, decode_pc) \ + DECODE_VALUE(label, decode_pc) -#define DECODE_LITERAL(val, code_chunk, base_index, off) \ - DECODE_VALUE(val, code_chunk, base_index, off) +#define DECODE_LITERAL(val, decode_pc) \ + DECODE_VALUE(val, decode_pc) -#define DECODE_INTEGER(integer, code_chunk, base_index, off) \ - DECODE_VALUE(integer, code_chunk, base_index, off) +#define DECODE_INTEGER(integer, decode_pc) \ + DECODE_VALUE(integer, decode_pc) -#define DECODE_XREG(reg, code_chunk, base_index, off) \ - DECODE_VALUE(reg, code_chunk, base_index, off) +#define DECODE_XREG(reg, decode_pc) \ + DECODE_VALUE(reg, decode_pc) -#define DECODE_YREG(reg, code_chunk, base_index, off) \ - DECODE_VALUE(reg, code_chunk, base_index, off) +#define DECODE_YREG(reg, decode_pc) \ + DECODE_VALUE(reg, decode_pc) -#define DECODE_ALLOCATOR_LIST(need, code_chunk, base_index, off) \ - if (IS_EXTENDED_ALLOCATOR(code_chunk, base_index, off)) { \ +#define DECODE_ALLOCATOR_LIST(need, decode_pc) \ + if (IS_EXTENDED_ALLOCATOR(decode_pc)) { \ need = 0; \ - off++; /* skip list tag */ \ + (decode_pc)++; /* skip list tag */ \ uint32_t list_size; \ - DECODE_LITERAL(list_size, code_chunk, base_index, off); \ + DECODE_LITERAL(list_size, decode_pc); \ uint32_t allocator_tag; \ uint32_t allocator_size; \ for (uint32_t j = 0; j < list_size; j++) { \ - DECODE_LITERAL(allocator_tag, code_chunk, base_index, off); \ - DECODE_LITERAL(allocator_size, code_chunk, base_index, off); \ + DECODE_LITERAL(allocator_tag, decode_pc); \ + DECODE_LITERAL(allocator_size, decode_pc); \ if (allocator_tag == COMPACT_EXTENDED_ALLOCATOR_LIST_TAG_FLOATS) { \ allocator_size *= FLOAT_SIZE; \ } else if (allocator_tag == COMPACT_EXTENDED_ALLOCATOR_LIST_TAG_FUNS) { \ @@ -759,19 +722,16 @@ typedef struct need += allocator_size; \ } \ } else { \ - DECODE_LITERAL(need, code_chunk, base_index, off); \ + DECODE_LITERAL(need, decode_pc); \ } #endif -#define IS_EXTENDED_ALLOCATOR(code_chunk, base_index, off) \ - (code_chunk[(base_index) + (off)]) == COMPACT_EXTENDED_ALLOCATION_LIST +#define IS_EXTENDED_ALLOCATOR(decode_pc) \ + (*decode_pc) == COMPACT_EXTENDED_ALLOCATION_LIST -#define IS_EXTENDED_FP_REGISTER(code_chunk, base_index, off) \ - (code_chunk[(base_index) + (off)]) == COMPACT_EXTENDED_FP_REGISTER - -#define NEXT_INSTRUCTION(operands_size) \ - i += operands_size +#define IS_EXTENDED_FP_REGISTER(decode_pc) \ + (*decode_pc) == COMPACT_EXTENDED_FP_REGISTER #define JUMP_TO_LABEL(module, label) \ if (module != mod) { \ @@ -783,11 +743,11 @@ typedef struct #ifndef TRACE_JUMP #define JUMP_TO_ADDRESS(address) \ - i = ((uint8_t *) (address)) - code + pc = (address) #else #define JUMP_TO_ADDRESS(address) \ - i = ((uint8_t *) (address)) - code; \ - fprintf(stderr, "going to jump to %i\n", i) + pc = (address); \ + fprintf(stderr, "going to jump to %" PRIuPTR "\n", (uintptr_t) (pc - code)) #endif #define SCHEDULE_NEXT(restore_mod, restore_to) \ @@ -864,7 +824,7 @@ typedef struct goto *next_label; \ } \ if (context_get_flags(ctx, Trap)) { \ - SCHEDULE_NEXT(mod, INSTRUCTION_POINTER()); \ + SCHEDULE_NEXT(mod, pc); \ } \ } @@ -873,18 +833,18 @@ typedef struct if (UNLIKELY(!context_get_flags(ctx, Trap))) { \ HANDLE_ERROR(); \ } else { \ - SCHEDULE_NEXT(mod, INSTRUCTION_POINTER()); \ + SCHEDULE_NEXT(mod, pc); \ } \ } -#define PROCESS_MAYBE_TRAP_RETURN_VALUE_RESTORE_I(return_value, rest_i) \ - if (term_is_invalid_term(return_value)) { \ - if (UNLIKELY(!context_get_flags(ctx, Trap))) { \ - i = rest_i; \ - HANDLE_ERROR(); \ - } else { \ - SCHEDULE_NEXT(mod, INSTRUCTION_POINTER()); \ - } \ +#define PROCESS_MAYBE_TRAP_RETURN_VALUE_RESTORE_PC(return_value, rest_pc) \ + if (term_is_invalid_term(return_value)) { \ + if (UNLIKELY(!context_get_flags(ctx, Trap))) { \ + pc = rest_pc; \ + HANDLE_ERROR(); \ + } else { \ + SCHEDULE_NEXT(mod, pc); \ + } \ } #define PROCESS_MAYBE_TRAP_RETURN_VALUE_LAST(return_value) \ @@ -893,13 +853,10 @@ typedef struct HANDLE_ERROR(); \ } else { \ DO_RETURN(); \ - SCHEDULE_NEXT(mod, INSTRUCTION_POINTER()); \ + SCHEDULE_NEXT(mod, pc); \ } \ } -#define INSTRUCTION_POINTER() \ - ((const void *) &code[i]) - #define DO_RETURN() \ { \ int module_index = ctx->cp >> 24; \ @@ -913,14 +870,11 @@ typedef struct mod = globalcontext_get_module_by_index(glb, module_index); \ code = mod->code->code; \ } \ - i = (ctx->cp & 0xFFFFFF) >> 2; \ + pc = code + ((ctx->cp & 0xFFFFFF) >> 2); \ } -#define POINTER_TO_II(instruction_pointer) \ - (((uint8_t *) (instruction_pointer)) - code) - -#define HANDLE_ERROR() \ - x_regs[2] = stacktrace_create_raw(ctx, mod, i, x_regs[0]); \ +#define HANDLE_ERROR() \ + x_regs[2] = stacktrace_create_raw(ctx, mod, pc - code, x_regs[0]); \ goto handle_error; #define VERIFY_IS_INTEGER(t, opcode_name) \ @@ -953,7 +907,7 @@ typedef struct RAISE_ERROR(BADARG_ATOM); \ } -#define CALL_FUN(fun, args_count, next_off) \ +#define CALL_FUN(fun, args_count) \ Module *fun_module; \ unsigned int fun_arity; \ uint32_t n_freeze = 0; \ @@ -968,7 +922,6 @@ typedef struct struct Nif *nif = (struct Nif *) nifs_get(module_name, function_name, fun_arity); \ if (!IS_NULL_PTR(nif)) { \ term return_value = nif->nif_ptr(ctx, fun_arity, x_regs); \ - NEXT_INSTRUCTION(next_off); \ PROCESS_MAYBE_TRAP_RETURN_VALUE(return_value); \ x_regs[0] = return_value; \ if (ctx->heap.root->next) { \ @@ -1001,8 +954,7 @@ typedef struct for (uint32_t i = 0; i < n_freeze; i++) { \ x_regs[i + fun_arity] = boxed_value[i + 3]; \ } \ - NEXT_INSTRUCTION(next_off); \ - ctx->cp = module_address(mod->module_index, i); \ + ctx->cp = module_address(mod->module_index, pc - code); \ JUMP_TO_LABEL(fun_module, label); #define DECODE_FLAGS_LIST(flags_value, flags, opcode) \ @@ -1133,7 +1085,7 @@ COLD_FUNC static void cp_to_mod_lbl_off(term cp, Context *ctx, Module **cp_mod, int labels_count = ENDIAN_SWAP_32(mod->code->labels); int i = 1; - uint8_t *l = mod->labels[1]; + const uint8_t *l = mod->labels[1]; while (mod_offset > l - code) { i++; if (i >= labels_count) { @@ -1146,7 +1098,7 @@ COLD_FUNC static void cp_to_mod_lbl_off(term cp, Context *ctx, Module **cp_mod, } *label = i - 1; - *l_off = mod_offset - ((uint8_t *) mod->labels[*label] - code); + *l_off = mod_offset - (mod->labels[*label] - code); } COLD_FUNC static void dump(Context *ctx) @@ -1268,70 +1220,61 @@ static inline term maybe_alloc_boxed_integer_fragment_helper(Context *ctx, avm_i } } -static term large_integer_to_term(Context *ctx, uint8_t *compact_term, int *next_operand_offset) +static term large_integer_to_term(Context *ctx, int num_bytes, const uint8_t *compact_term) { - int num_bytes = (*compact_term >> 5) + 2; - switch (num_bytes) { case 2: { - *next_operand_offset += 3; - int16_t ret_val16 = ((int16_t) compact_term[1]) << 8 | compact_term[2]; + int16_t ret_val16 = ((int16_t) compact_term[0]) << 8 | compact_term[1]; return maybe_alloc_boxed_integer_fragment_helper(ctx, ret_val16, 2); } case 3: { - *next_operand_offset += 4; struct Int24 ret_val24; - ret_val24.val24 = ((int32_t) compact_term[1]) << 16 | ((int32_t) compact_term[2] << 8) | compact_term[3]; + ret_val24.val24 = ((int32_t) compact_term[0]) << 16 | ((int32_t) compact_term[1] << 8) | compact_term[2]; return maybe_alloc_boxed_integer_fragment_helper(ctx, ret_val24.val24, 3); } case 4: { - *next_operand_offset += 5; int32_t ret_val32; - ret_val32 = ((int32_t) compact_term[1]) << 24 | ((int32_t) compact_term[2] << 16) - | ((int32_t) compact_term[3] << 8) | compact_term[4]; + ret_val32 = ((int32_t) compact_term[0]) << 24 | ((int32_t) compact_term[1] << 16) + | ((int32_t) compact_term[2] << 8) | compact_term[3]; return maybe_alloc_boxed_integer_fragment_helper(ctx, ret_val32, 4); } case 5: { - *next_operand_offset += 6; struct Int40 ret_val40; - ret_val40.val40 = ((int64_t) compact_term[1]) << 32 | ((int64_t) compact_term[2] << 24) - | ((int64_t) compact_term[3] << 16) | ((int64_t) compact_term[4] << 8) - | (int64_t) compact_term[5]; + ret_val40.val40 = ((int64_t) compact_term[0]) << 32 | ((int64_t) compact_term[1] << 24) + | ((int64_t) compact_term[2] << 16) | ((int64_t) compact_term[3] << 8) + | (int64_t) compact_term[4]; return maybe_alloc_boxed_integer_fragment_helper(ctx, ret_val40.val40, 5); } case 6: { - *next_operand_offset += 7; struct Int48 ret_val48; - ret_val48.val48 = ((int64_t) compact_term[1]) << 40 | ((int64_t) compact_term[2] << 32) - | ((int64_t) compact_term[3] << 24) | ((int64_t) compact_term[4] << 16) - | ((int64_t) compact_term[5] << 8) | (int64_t) compact_term[6]; + ret_val48.val48 = ((int64_t) compact_term[0]) << 40 | ((int64_t) compact_term[1] << 32) + | ((int64_t) compact_term[2] << 24) | ((int64_t) compact_term[3] << 16) + | ((int64_t) compact_term[4] << 8) | (int64_t) compact_term[5]; return maybe_alloc_boxed_integer_fragment_helper(ctx, ret_val48.val48, 6); } case 7: { - *next_operand_offset += 8; struct Int56 ret_val56; - ret_val56.val56 = ((int64_t) compact_term[1]) << 48 | ((int64_t) compact_term[2] << 40) - | ((int64_t) compact_term[3] << 32) | ((int64_t) compact_term[4] << 24) - | ((int64_t) compact_term[5] << 16) | ((int64_t) compact_term[6] << 8) - | (int64_t) compact_term[7]; + ret_val56.val56 = ((int64_t) compact_term[0]) << 48 | ((int64_t) compact_term[1] << 40) + | ((int64_t) compact_term[2] << 32) | ((int64_t) compact_term[3] << 24) + | ((int64_t) compact_term[4] << 16) | ((int64_t) compact_term[5] << 8) + | (int64_t) compact_term[6]; return maybe_alloc_boxed_integer_fragment_helper(ctx, ret_val56.val56, 7); } case 8: { - *next_operand_offset += 9; int64_t ret_val64; - ret_val64 = ((int64_t) compact_term[1]) << 56 | ((int64_t) compact_term[2] << 48) - | ((int64_t) compact_term[3] << 40) | ((int64_t) compact_term[4] << 32) - | ((int64_t) compact_term[5] << 24) | ((int64_t) compact_term[6] << 16) - | ((int64_t) compact_term[7] << 8) | (int64_t) compact_term[8]; + ret_val64 = ((int64_t) compact_term[0]) << 56 | ((int64_t) compact_term[1] << 48) + | ((int64_t) compact_term[2] << 40) | ((int64_t) compact_term[3] << 32) + | ((int64_t) compact_term[4] << 24) | ((int64_t) compact_term[5] << 16) + | ((int64_t) compact_term[6] << 8) | (int64_t) compact_term[7]; return maybe_alloc_boxed_integer_fragment_helper(ctx, ret_val64, 8); } @@ -1584,11 +1527,11 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif { #ifdef IMPL_EXECUTE_LOOP - uint8_t *code; + const uint8_t *code; Module *mod; Module *prev_mod; term *x_regs; - uintptr_t i; + const uint8_t *pc; int remaining_reductions; Context *ctx = scheduler_run(glb); @@ -1617,39 +1560,38 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #ifdef IMPL_CODE_LOADER TRACE("-- Loading code\n"); SMP_MODULE_LOCK(mod); - uint8_t *code = mod->code->code; - uintptr_t i = 0; + const uint8_t *code = mod->code->code; + const uint8_t *pc = code; #endif while (1) { - TRACE("-- loop -- i = %d\n", (int) i); + TRACE("-- loop -- i = %" PRIuPTR ", next ocopde = %d\n", pc - code, *pc); - switch (code[i]) { + switch (*pc++) { case OP_LABEL: { + #ifdef IMPL_CODE_LOADER + const uint8_t* saved_pc = pc - 1; + #endif uint32_t label; - int next_off = 1; - DECODE_LITERAL(label, code, i, next_off) + DECODE_LITERAL(label, pc) TRACE("label/1 label=%i\n", label); USED_BY_TRACE(label); #ifdef IMPL_CODE_LOADER - TRACE("Mark label %i here at %i\n", label, (int) i); - module_add_label(mod, label, &code[i]); + TRACE("Mark label %i here at %" PRIuPTR "\n", label, saved_pc - code); + module_add_label(mod, label, saved_pc); #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_FUNC_INFO: { - int next_off = 1; int module_atom; - DECODE_ATOM(module_atom, code, i, next_off) + DECODE_ATOM(module_atom, pc) int function_name_atom; - DECODE_ATOM(function_name_atom, code, i, next_off) + DECODE_ATOM(function_name_atom, pc) uint32_t arity; - DECODE_LITERAL(arity, code, i, next_off); + DECODE_LITERAL(arity, pc); TRACE("func_info/3 module_name_a=%i, function_name_a=%i, arity=%i\n", module_atom, function_name_atom, arity); USED_BY_TRACE(function_name_atom); @@ -1659,8 +1601,6 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #ifdef IMPL_EXECUTE_LOOP RAISE_ERROR(FUNCTION_CLAUSE_ATOM); #endif - - NEXT_INSTRUCTION(next_off); break; } @@ -1670,7 +1610,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #ifdef IMPL_CODE_LOADER TRACE("-- Code loading finished --\n"); SMP_MODULE_UNLOCK(mod); - return i; + return pc - code - 1; #endif #ifdef IMPL_EXECUTE_LOOP @@ -1679,18 +1619,16 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } case OP_CALL: { - int next_off = 1; uint32_t arity; - DECODE_LITERAL(arity, code, i, next_off); + DECODE_LITERAL(arity, pc); uint32_t label; - DECODE_LABEL(label, code, i, next_off); + DECODE_LABEL(label, pc); TRACE("call/2, arity=%i, label=%i\n", arity, label); USED_BY_TRACE(arity); #ifdef IMPL_EXECUTE_LOOP - NEXT_INSTRUCTION(next_off); - ctx->cp = module_address(mod->module_index, i); + ctx->cp = module_address(mod->module_index, pc - code); remaining_reductions--; if (LIKELY(remaining_reductions)) { @@ -1701,21 +1639,16 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } #endif - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); - #endif - break; } case OP_CALL_LAST: { - int next_off = 1; uint32_t arity; - DECODE_LITERAL(arity, code, i, next_off); + DECODE_LITERAL(arity, pc); uint32_t label; - DECODE_LABEL(label, code, i, next_off); + DECODE_LABEL(label, pc); uint32_t n_words; - DECODE_LITERAL(n_words, code, i, next_off); + DECODE_LITERAL(n_words, pc); TRACE("call_last/3, arity=%i, label=%i, dellocate=%i\n", arity, label, n_words); USED_BY_TRACE(arity); @@ -1736,28 +1669,20 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) SCHEDULE_NEXT(mod, mod->labels[label]); } #endif - - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); - #endif - break; } case OP_CALL_ONLY: { - int next_off = 1; uint32_t arity; - DECODE_LITERAL(arity, code, i, next_off); + DECODE_LITERAL(arity, pc); uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) TRACE("call_only/2, arity=%i, label=%i\n", arity, label); USED_BY_TRACE(arity); USED_BY_TRACE(label); #ifdef IMPL_EXECUTE_LOOP - - NEXT_INSTRUCTION(next_off); remaining_reductions--; if (LIKELY(remaining_reductions)) { TRACE_CALL(ctx, mod, "call_only", label, arity); @@ -1766,40 +1691,30 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) SCHEDULE_NEXT(mod, mod->labels[label]); } #endif - - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); - #endif - break; } case OP_CALL_EXT: { - int next_off = 1; + #ifdef IMPL_EXECUTE_LOOP + // save pc in case of error + const uint8_t *orig_pc = pc - 1; + + remaining_reductions--; + if (UNLIKELY(!remaining_reductions)) { + SCHEDULE_NEXT(mod, orig_pc); + } + #endif uint32_t arity; - DECODE_LITERAL(arity, code, i, next_off); + DECODE_LITERAL(arity, pc); uint32_t index; - DECODE_LITERAL(index, code, i, next_off); + DECODE_LITERAL(index, pc); TRACE("call_ext/2, arity=%i, index=%i\n", arity, index); USED_BY_TRACE(arity); USED_BY_TRACE(index); - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); - #endif - #ifdef IMPL_EXECUTE_LOOP - remaining_reductions--; - if (UNLIKELY(!remaining_reductions)) { - SCHEDULE_NEXT(mod, INSTRUCTION_POINTER()); - } - - // save instruction offset in case of error - int orig_i = i; - NEXT_INSTRUCTION(next_off); - TRACE_CALL_EXT(ctx, mod, "call_ext", index, arity); const struct ExportedFunction *func = module_resolve_function(mod, index, glb); @@ -1811,7 +1726,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) case NIFFunctionType: { const struct Nif *nif = EXPORTED_FUNCTION_TO_NIF(func); term return_value = nif->nif_ptr(ctx, arity, x_regs); - PROCESS_MAYBE_TRAP_RETURN_VALUE_RESTORE_I(return_value, orig_i); + PROCESS_MAYBE_TRAP_RETURN_VALUE_RESTORE_PC(return_value, orig_pc); x_regs[0] = return_value; if (ctx->heap.root->next) { if (UNLIKELY(memory_ensure_free_opt(ctx, 0, MEMORY_FORCE_SHRINK) != MEMORY_GC_OK)) { @@ -1823,7 +1738,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) case ModuleFunction: { const struct ModuleFunction *jump = EXPORTED_FUNCTION_TO_MODULE_FUNCTION(func); - ctx->cp = module_address(mod->module_index, i); + ctx->cp = module_address(mod->module_index, pc - code); JUMP_TO_LABEL(jump->target, jump->label); break; @@ -1859,13 +1774,21 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } case OP_CALL_EXT_LAST: { - int next_off = 1; + #ifdef IMPL_EXECUTE_LOOP + // save pc in case of error + const uint8_t *orig_pc = pc - 1; + + remaining_reductions--; + if (UNLIKELY(!remaining_reductions)) { + SCHEDULE_NEXT(mod, orig_pc); + } + #endif uint32_t arity; - DECODE_LITERAL(arity, code, i, next_off); + DECODE_LITERAL(arity, pc); uint32_t index; - DECODE_LITERAL(index, code, i, next_off); + DECODE_LITERAL(index, pc); uint32_t n_words; - DECODE_LITERAL(n_words, code, i, next_off); + DECODE_LITERAL(n_words, pc); TRACE("call_ext_last/3, arity=%i, index=%i, n_words=%i\n", arity, index, n_words); USED_BY_TRACE(arity); @@ -1873,11 +1796,6 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) USED_BY_TRACE(n_words); #ifdef IMPL_EXECUTE_LOOP - remaining_reductions--; - if (UNLIKELY(!remaining_reductions)) { - SCHEDULE_NEXT(mod, INSTRUCTION_POINTER()); - } - TRACE_CALL_EXT(ctx, mod, "call_ext_last", index, arity); const struct ExportedFunction *func = module_resolve_function(mod, index, glb); @@ -1953,20 +1871,14 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } } #endif - - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); - #endif - break; } case OP_BIF0: { - int next_off = 1; uint32_t bif; - DECODE_LITERAL(bif, code, i, next_off); + DECODE_LITERAL(bif, pc); dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); TRACE("bif0/2 bif=%i, dreg=%c%i\n", bif, T_DEST_REG(dreg)); USED_BY_TRACE(bif); @@ -1980,22 +1892,19 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) WRITE_REGISTER(dreg, ret); #endif - - NEXT_INSTRUCTION(next_off); break; } //TODO: implement me case OP_BIF1: { - int next_off = 1; uint32_t fail_label; - DECODE_LABEL(fail_label, code, i, next_off); + DECODE_LABEL(fail_label, pc); uint32_t bif; - DECODE_LITERAL(bif, code, i, next_off); + DECODE_LITERAL(bif, pc); term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); TRACE("bif1/2 bif=%i, fail=%i, dreg=%c%i\n", bif, fail_label, T_DEST_REG(dreg)); USED_BY_TRACE(bif); @@ -2016,24 +1925,21 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) WRITE_REGISTER(dreg, ret); #endif - - NEXT_INSTRUCTION(next_off); break; } //TODO: implement me case OP_BIF2: { - int next_off = 1; uint32_t fail_label; - DECODE_LABEL(fail_label, code, i, next_off); + DECODE_LABEL(fail_label, pc); uint32_t bif; - DECODE_LITERAL(bif, code, i, next_off); + DECODE_LITERAL(bif, pc); term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) term arg2; - DECODE_COMPACT_TERM(arg2, code, i, next_off) + DECODE_COMPACT_TERM(arg2, pc) dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); TRACE("bif2/2 bif=%i, fail=%i, dreg=%c%i\n", bif, fail_label, T_DEST_REG(dreg)); USED_BY_TRACE(bif); @@ -2055,17 +1961,14 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) WRITE_REGISTER(dreg, ret); #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_ALLOCATE: { - int next_off = 1; uint32_t stack_need; - DECODE_LITERAL(stack_need, code, i, next_off); + DECODE_LITERAL(stack_need, pc); uint32_t live; - DECODE_LITERAL(live, code, i, next_off); + DECODE_LITERAL(live, pc); TRACE("allocate/2 stack_need=%i, live=%i\n" , stack_need, live); USED_BY_TRACE(stack_need); USED_BY_TRACE(live); @@ -2088,19 +1991,16 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) ctx->e -= stack_need + 1; ctx->e[stack_need] = ctx->cp; #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_ALLOCATE_HEAP: { - int next_off = 1; uint32_t stack_need; - DECODE_LITERAL(stack_need, code, i, next_off); + DECODE_LITERAL(stack_need, pc); uint32_t heap_need; - DECODE_ALLOCATOR_LIST(heap_need, code, i, next_off); + DECODE_ALLOCATOR_LIST(heap_need, pc); uint32_t live; - DECODE_LITERAL(live, code, i, next_off); + DECODE_LITERAL(live, pc); TRACE("allocate_heap/2 stack_need=%i, heap_need=%i, live=%i\n", stack_need, heap_need, live); USED_BY_TRACE(stack_need); USED_BY_TRACE(heap_need); @@ -2124,17 +2024,14 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) ctx->e -= stack_need + 1; ctx->e[stack_need] = ctx->cp; #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_ALLOCATE_ZERO: { - int next_off = 1; uint32_t stack_need; - DECODE_LITERAL(stack_need, code, i, next_off); + DECODE_LITERAL(stack_need, pc); uint32_t live; - DECODE_LITERAL(live, code, i, next_off); + DECODE_LITERAL(live, pc); TRACE("allocate_zero/2 stack_need=%i, live=%i\n", stack_need, live); USED_BY_TRACE(stack_need); USED_BY_TRACE(live); @@ -2161,19 +2058,16 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } ctx->e[stack_need] = ctx->cp; #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_ALLOCATE_HEAP_ZERO: { - int next_off = 1; uint32_t stack_need; - DECODE_LITERAL(stack_need, code, i, next_off); + DECODE_LITERAL(stack_need, pc); uint32_t heap_need; - DECODE_ALLOCATOR_LIST(heap_need, code, i, next_off); + DECODE_ALLOCATOR_LIST(heap_need, pc); uint32_t live; - DECODE_LITERAL(live, code, i, next_off); + DECODE_LITERAL(live, pc); TRACE("allocate_heap_zero/3 stack_need=%i, heap_need=%i, live=%i\n", stack_need, heap_need, live); USED_BY_TRACE(stack_need); USED_BY_TRACE(heap_need); @@ -2200,17 +2094,14 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } ctx->e[stack_need] = ctx->cp; #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_TEST_HEAP: { - int next_off = 1; uint32_t heap_need; - DECODE_ALLOCATOR_LIST(heap_need, code, i, next_off); + DECODE_ALLOCATOR_LIST(heap_need, pc); uint32_t live_registers; - DECODE_LITERAL(live_registers, code, i, next_off); + DECODE_LITERAL(live_registers, pc); TRACE("test_heap/2 heap_need=%i, live_registers=%i\n", heap_need, live_registers); USED_BY_TRACE(heap_need); @@ -2234,31 +2125,24 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } } #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_KILL: { - int next_off = 1; uint32_t target; - DECODE_YREG(target, code, i, next_off); + DECODE_YREG(target, pc); TRACE("kill/1 target=%i\n", target); #ifdef IMPL_EXECUTE_LOOP ctx->e[target] = term_nil(); #endif - - NEXT_INSTRUCTION(next_off); - break; } case OP_DEALLOCATE: { - int next_off = 1; uint32_t n_words; - DECODE_LITERAL(n_words, code, i, next_off); + DECODE_LITERAL(n_words, pc); TRACE("deallocate/1 n_words=%i\n", n_words); USED_BY_TRACE(n_words); @@ -2275,8 +2159,6 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } } #endif - - NEXT_INSTRUCTION(next_off); break; } @@ -2292,10 +2174,6 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) DO_RETURN(); #endif - - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(1); - #endif break; } @@ -2313,8 +2191,6 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) x_regs[0] = x_regs[1]; #endif - - NEXT_INSTRUCTION(1); break; } @@ -2332,8 +2208,6 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) mailbox_remove_message(&ctx->mailbox, &ctx->heap); // Cannot GC now as remove_message is GC neutral #endif - - NEXT_INSTRUCTION(1); break; } @@ -2345,17 +2219,14 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) mailbox_reset(&ctx->mailbox); #endif - - NEXT_INSTRUCTION(1); break; } case OP_LOOP_REC: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); TRACE("loop_rec/2, dreg=%c%i\n", T_DEST_REG(dreg)); USED_BY_TRACE(dreg); @@ -2370,23 +2241,16 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE_RECEIVE(ctx, ret); WRITE_REGISTER(dreg, ret); - NEXT_INSTRUCTION(next_off); } else { JUMP_TO_ADDRESS(mod->labels[label]); } #endif - - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); - #endif - break; } case OP_LOOP_REC_END: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off); + DECODE_LABEL(label, pc); TRACE("loop_rec_end/1 label=%i\n", label); USED_BY_TRACE(label); @@ -2397,18 +2261,15 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) PROCESS_SIGNAL_MESSAGES(); #pragma GCC diagnostic pop mailbox_next(&ctx->mailbox); - i = POINTER_TO_II(mod->labels[label]); -#else - NEXT_INSTRUCTION(next_off); + pc = mod->labels[label]; #endif break; } //TODO: implement wait/1 case OP_WAIT: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) TRACE("wait/1\n"); @@ -2422,21 +2283,19 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) ctx = scheduler_wait(ctx); goto schedule_in; #endif - - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); - #endif - break; } //TODO: implement wait_timeout/2 case OP_WAIT_TIMEOUT: { - int next_off = 1; + #ifdef IMPL_EXECUTE_LOOP + // PC for wait_timeout_trap_handler, just before label + const uint8_t *saved_pc = pc; + #endif uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term timeout; - DECODE_COMPACT_TERM(timeout, code, i, next_off) + DECODE_COMPACT_TERM(timeout, pc) #ifdef IMPL_EXECUTE_LOOP avm_int64_t t = 0; @@ -2467,7 +2326,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } if (needs_to_wait) { - ctx->saved_ip = INSTRUCTION_POINTER(); + ctx->saved_ip = saved_pc; #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wpedantic" ctx->restore_trap_handler = &&wait_timeout_trap_handler; @@ -2475,8 +2334,6 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) ctx->saved_module = mod; ctx = scheduler_wait(ctx); goto schedule_in; - } else { - NEXT_INSTRUCTION(next_off); } #endif @@ -2484,8 +2341,6 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("wait_timeout/2, label: %i\n", label); UNUSED(timeout) - - NEXT_INSTRUCTION(next_off); #endif break; @@ -2497,11 +2352,10 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) // Determine if a message arrived to either jump to timeout label // or to continuation. // Redo the offset computation and refetch the label - int next_off = 1; int label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) int timeout; - DECODE_COMPACT_TERM(timeout, code, i, next_off) + DECODE_COMPACT_TERM(timeout, pc) TRACE("wait_timeout_trap_handler, label: %i\n", label); #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wpedantic" @@ -2509,7 +2363,6 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #pragma GCC diagnostic pop if (context_get_flags(ctx, WaitingTimeoutExpired)) { ctx->restore_trap_handler = NULL; - NEXT_INSTRUCTION(next_off); } else { if (UNLIKELY(!mailbox_has_next(&ctx->mailbox))) { // No message is here. @@ -2526,23 +2379,20 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif case OP_IS_LT: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off); + DECODE_LABEL(label, pc); term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off); + DECODE_COMPACT_TERM(arg1, pc); term arg2; - DECODE_COMPACT_TERM(arg2, code, i, next_off); + DECODE_COMPACT_TERM(arg2, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("is_lt/2, label=%i, arg1=%lx, arg2=%lx\n", label, arg1, arg2); TermCompareResult result = term_compare(arg1, arg2, TermCompareNoOpts, ctx->global); - if (result == TermLessThan) { - NEXT_INSTRUCTION(next_off); - } else if (result & (TermGreaterThan | TermEquals)) { - i = POINTER_TO_II(mod->labels[label]); - } else { + if (result & (TermGreaterThan | TermEquals)) { + pc = mod->labels[label]; + } else if (UNLIKELY(result == TermCompareMemoryAllocFail)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } #endif @@ -2551,30 +2401,26 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("is_lt/2\n"); UNUSED(arg1) UNUSED(arg2) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_IS_GE: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off); + DECODE_LABEL(label, pc); term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off); + DECODE_COMPACT_TERM(arg1, pc); term arg2; - DECODE_COMPACT_TERM(arg2, code, i, next_off); + DECODE_COMPACT_TERM(arg2, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("is_ge/2, label=%i, arg1=%lx, arg2=%lx\n", label, arg1, arg2); TermCompareResult result = term_compare(arg1, arg2, TermCompareNoOpts, ctx->global); - if (result & (TermGreaterThan | TermEquals)) { - NEXT_INSTRUCTION(next_off); - } else if (result == TermLessThan) { - i = POINTER_TO_II(mod->labels[label]); - } else { + if (result == TermLessThan) { + pc = mod->labels[label]; + } else if (UNLIKELY(result == TermCompareMemoryAllocFail)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } #endif @@ -2583,30 +2429,26 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("is_ge/2\n"); UNUSED(arg1) UNUSED(arg2) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_IS_EQUAL: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) term arg2; - DECODE_COMPACT_TERM(arg2, code, i, next_off) + DECODE_COMPACT_TERM(arg2, pc) #ifdef IMPL_EXECUTE_LOOP TRACE("is_equal/2, label=%i, arg1=%lx, arg2=%lx\n", label, arg1, arg2); TermCompareResult result = term_compare(arg1, arg2, TermCompareNoOpts, ctx->global); - if (result == TermEquals) { - NEXT_INSTRUCTION(next_off); - } else if (result & (TermLessThan | TermGreaterThan)) { - i = POINTER_TO_II(mod->labels[label]); - } else { + if (result & (TermLessThan | TermGreaterThan)) { + pc = mod->labels[label]; + } else if (UNLIKELY(result == TermCompareMemoryAllocFail)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } #endif @@ -2615,30 +2457,26 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("is_equal/2\n"); UNUSED(arg1) UNUSED(arg2) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_IS_NOT_EQUAL: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) term arg2; - DECODE_COMPACT_TERM(arg2, code, i, next_off) + DECODE_COMPACT_TERM(arg2, pc) #ifdef IMPL_EXECUTE_LOOP TRACE("is_not_equal/2, label=%i, arg1=%lx, arg2=%lx\n", label, arg1, arg2); TermCompareResult result = term_compare(arg1, arg2, TermCompareNoOpts, ctx->global); - if (result & (TermLessThan | TermGreaterThan)) { - NEXT_INSTRUCTION(next_off); - } else if (result == TermEquals) { - i = POINTER_TO_II(mod->labels[label]); - } else { + if (result == TermEquals) { + pc = mod->labels[label]; + } else if (UNLIKELY(result == TermCompareMemoryAllocFail)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } #endif @@ -2647,30 +2485,26 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("is_not_equal/2\n"); UNUSED(arg1) UNUSED(arg2) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_IS_EQ_EXACT: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) term arg2; - DECODE_COMPACT_TERM(arg2, code, i, next_off) + DECODE_COMPACT_TERM(arg2, pc) #ifdef IMPL_EXECUTE_LOOP TRACE("is_eq_exact/2, label=%i, arg1=%lx, arg2=%lx\n", label, arg1, arg2); TermCompareResult result = term_compare(arg1, arg2, TermCompareExact, ctx->global); - if (result == TermEquals) { - NEXT_INSTRUCTION(next_off); - } else if (result & (TermLessThan | TermGreaterThan)) { - i = POINTER_TO_II(mod->labels[label]); - } else { + if (result & (TermLessThan | TermGreaterThan)) { + pc = mod->labels[label]; + } else if (UNLIKELY(result == TermCompareMemoryAllocFail)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } #endif @@ -2679,30 +2513,26 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("is_eq_exact/2\n"); UNUSED(arg1) UNUSED(arg2) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_IS_NOT_EQ_EXACT: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) term arg2; - DECODE_COMPACT_TERM(arg2, code, i, next_off) + DECODE_COMPACT_TERM(arg2, pc) #ifdef IMPL_EXECUTE_LOOP TRACE("is_not_eq_exact/2, label=%i, arg1=%lx, arg2=%lx\n", label, arg1, arg2); TermCompareResult result = term_compare(arg1, arg2, TermCompareExact, ctx->global); - if (result & (TermLessThan | TermGreaterThan)) { - NEXT_INSTRUCTION(next_off); - } else if (result == TermEquals) { - i = POINTER_TO_II(mod->labels[label]); - } else { + if (result == TermEquals) { + pc = mod->labels[label]; + } else if (UNLIKELY(result == TermCompareMemoryAllocFail)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } #endif @@ -2711,26 +2541,22 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("is_not_eq_exact/2\n"); UNUSED(arg1) UNUSED(arg2) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_IS_INTEGER: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) #ifdef IMPL_EXECUTE_LOOP TRACE("is_integer/2, label=%i, arg1=%lx\n", label, arg1); - if (term_is_any_integer(arg1)) { - NEXT_INSTRUCTION(next_off); - } else { - i = POINTER_TO_II(mod->labels[label]); + if (!term_is_any_integer(arg1)) { + pc = mod->labels[label]; } #endif @@ -2738,26 +2564,22 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("is_integer/2\n"); UNUSED(label) UNUSED(arg1) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_IS_FLOAT: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) #ifdef IMPL_EXECUTE_LOOP TRACE("is_float/2, label=%i, arg1=%lx\n", label, arg1); - if (term_is_float(arg1)) { - NEXT_INSTRUCTION(next_off); - } else { - i = POINTER_TO_II(mod->labels[label]); + if (!term_is_float(arg1)) { + pc = mod->labels[label]; } #endif @@ -2765,27 +2587,23 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("is_float/2\n"); UNUSED(label) UNUSED(arg1) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_IS_NUMBER: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) #ifdef IMPL_EXECUTE_LOOP TRACE("is_number/2, label=%i, arg1=%lx\n", label, arg1); //TODO: check for floats too - if (term_is_number(arg1)) { - NEXT_INSTRUCTION(next_off); - } else { - i = POINTER_TO_II(mod->labels[label]); + if (!term_is_number(arg1)) { + pc = mod->labels[label]; } #endif @@ -2793,200 +2611,170 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("is_number/2\n"); UNUSED(label) UNUSED(arg1) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_IS_BINARY: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) #ifdef IMPL_EXECUTE_LOOP TRACE("is_binary/2, label=%i, arg1=%lx\n", label, arg1); - if (term_is_binary(arg1)) { - NEXT_INSTRUCTION(next_off); - } else { - i = POINTER_TO_II(mod->labels[label]); + if (!term_is_binary(arg1)) { + pc = mod->labels[label]; } #endif #ifdef IMPL_CODE_LOADER TRACE("is_binary/2\n"); UNUSED(arg1) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_IS_LIST: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) #ifdef IMPL_EXECUTE_LOOP TRACE("is_list/2, label=%i, arg1=%lx\n", label, arg1); - if (term_is_list(arg1)) { - NEXT_INSTRUCTION(next_off); - } else { - i = POINTER_TO_II(mod->labels[label]); + if (!term_is_list(arg1)) { + pc = mod->labels[label]; } #endif #ifdef IMPL_CODE_LOADER TRACE("is_list/2\n"); UNUSED(arg1) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_IS_NONEMPTY_LIST: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) #ifdef IMPL_EXECUTE_LOOP TRACE("is_nonempty_list/2, label=%i, arg1=%lx\n", label, arg1); - if (term_is_nonempty_list(arg1)) { - NEXT_INSTRUCTION(next_off); - } else { - i = POINTER_TO_II(mod->labels[label]); + if (!term_is_nonempty_list(arg1)) { + pc = mod->labels[label]; } #endif #ifdef IMPL_CODE_LOADER TRACE("is_nonempty_list/2\n"); UNUSED(arg1) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_IS_NIL: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) #ifdef IMPL_EXECUTE_LOOP TRACE("is_nil/2, label=%i, arg1=%lx\n", label, arg1); - if (term_is_nil(arg1)) { - NEXT_INSTRUCTION(next_off); - } else { - i = POINTER_TO_II(mod->labels[label]); + if (!term_is_nil(arg1)) { + pc = mod->labels[label]; } #endif #ifdef IMPL_CODE_LOADER TRACE("is_nil/2\n"); UNUSED(arg1) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_IS_ATOM: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) #ifdef IMPL_EXECUTE_LOOP TRACE("is_atom/2, label=%i, arg1=%lx\n", label, arg1); - if (term_is_atom(arg1)) { - NEXT_INSTRUCTION(next_off); - } else { - i = POINTER_TO_II(mod->labels[label]); + if (!term_is_atom(arg1)) { + pc = mod->labels[label]; } #endif #ifdef IMPL_CODE_LOADER TRACE("is_atom/2\n"); UNUSED(arg1) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_IS_PID: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) #ifdef IMPL_EXECUTE_LOOP TRACE("is_pid/2, label=%i, arg1=%lx\n", label, arg1); - if (term_is_pid(arg1)) { - NEXT_INSTRUCTION(next_off); - } else { - i = POINTER_TO_II(mod->labels[label]); + if (!term_is_pid(arg1)) { + pc = mod->labels[label]; } #endif #ifdef IMPL_CODE_LOADER TRACE("is_pid/2\n"); UNUSED(arg1) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_IS_REFERENCE: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) #ifdef IMPL_EXECUTE_LOOP TRACE("is_reference/2, label=%i, arg1=%lx\n", label, arg1); - if (term_is_reference(arg1)) { - NEXT_INSTRUCTION(next_off); - } else { - i = POINTER_TO_II(mod->labels[label]); + if (!term_is_reference(arg1)) { + pc = mod->labels[label]; } #endif #ifdef IMPL_CODE_LOADER TRACE("is_reference/2\n"); UNUSED(arg1) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_IS_PORT: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) #ifdef IMPL_EXECUTE_LOOP TRACE("is_port/2, label=%i, arg1=%lx\n", label, arg1); @@ -2999,39 +2787,33 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) is_port_driver = context_is_port_driver(target); globalcontext_get_process_unlock(ctx->global, target); } - if (is_port_driver) { - NEXT_INSTRUCTION(next_off); - } else { - i = POINTER_TO_II(mod->labels[label]); + if (!is_port_driver) { + pc = mod->labels[label]; } } else { - i = POINTER_TO_II(mod->labels[label]); + pc = mod->labels[label]; } #endif #ifdef IMPL_CODE_LOADER TRACE("is_port/2\n"); UNUSED(arg1) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_IS_TUPLE: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) #ifdef IMPL_EXECUTE_LOOP TRACE("is_tuple/2, label=%i, arg1=%lx\n", label, arg1); - if (term_is_tuple(arg1)) { - NEXT_INSTRUCTION(next_off); - } else { - i = POINTER_TO_II(mod->labels[label]); + if (!term_is_tuple(arg1)) { + pc = mod->labels[label]; } #endif @@ -3039,28 +2821,24 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("is_tuple/2\n"); UNUSED(label) UNUSED(arg1) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_TEST_ARITY: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off); + DECODE_LABEL(label, pc); term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off); + DECODE_COMPACT_TERM(arg1, pc); uint32_t arity; - DECODE_LITERAL(arity, code, i, next_off); + DECODE_LITERAL(arity, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("test_arity/2, label=%i, arg1=%lx\n", label, arg1); - if (term_is_tuple(arg1) && (uint32_t) term_get_tuple_arity(arg1) == arity) { - NEXT_INSTRUCTION(next_off); - } else { - i = (uint8_t *) mod->labels[label] - code; + if (!(term_is_tuple(arg1) && (uint32_t) term_get_tuple_arity(arg1) == arity)) { + pc = mod->labels[label]; } #endif @@ -3068,21 +2846,19 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("test_arity/2\n"); UNUSED(label) UNUSED(arg1) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_SELECT_VAL: { - int next_off = 1; term src_value; - DECODE_COMPACT_TERM(src_value, code, i, next_off) + DECODE_COMPACT_TERM(src_value, pc) uint32_t default_label; - DECODE_LABEL(default_label, code, i, next_off) - DECODE_EXTENDED_LIST_TAG(code, i, next_off); + DECODE_LABEL(default_label, pc) + DECODE_EXTENDED_LIST_TAG(pc); uint32_t size; - DECODE_LITERAL(size, code, i, next_off) + DECODE_LITERAL(size, pc) TRACE("select_val/3, default_label=%i, vals=%i\n", default_label, size); USED_BY_TRACE(default_label); @@ -3093,14 +2869,14 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif #ifdef IMPL_EXECUTE_LOOP - void *jump_to_address = NULL; + const uint8_t *jump_to_address = NULL; #endif for (uint32_t j = 0; j < size / 2; j++) { term cmp_value; - DECODE_COMPACT_TERM(cmp_value, code, i, next_off) + DECODE_COMPACT_TERM(cmp_value, pc) uint32_t jmp_label; - DECODE_LABEL(jmp_label, code, i, next_off) + DECODE_LABEL(jmp_label, pc) #ifdef IMPL_CODE_LOADER UNUSED(cmp_value); @@ -3120,23 +2896,17 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) JUMP_TO_ADDRESS(jump_to_address); } #endif - - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); - #endif - break; } case OP_SELECT_TUPLE_ARITY: { - int next_off = 1; term src_value; - DECODE_COMPACT_TERM(src_value, code, i, next_off) + DECODE_COMPACT_TERM(src_value, pc) uint32_t default_label; - DECODE_LABEL(default_label, code, i, next_off) - DECODE_EXTENDED_LIST_TAG(code, i, next_off); + DECODE_LABEL(default_label, pc) + DECODE_EXTENDED_LIST_TAG(pc); uint32_t size; - DECODE_LITERAL(size, code, i, next_off) + DECODE_LITERAL(size, pc) TRACE("select_tuple_arity/3, default_label=%i, vals=%i\n", default_label, size); USED_BY_TRACE(default_label); @@ -3147,7 +2917,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif #ifdef IMPL_EXECUTE_LOOP - void *jump_to_address = NULL; + const uint8_t *jump_to_address = NULL; #endif #ifdef IMPL_EXECUTE_LOOP @@ -3157,9 +2927,9 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) for (uint32_t j = 0; j < size / 2; j++) { uint32_t cmp_value; - DECODE_LITERAL(cmp_value, code, i, next_off) + DECODE_LITERAL(cmp_value, pc) uint32_t jmp_label; - DECODE_LABEL(jmp_label, code, i, next_off) + DECODE_LABEL(jmp_label, pc) #ifdef IMPL_CODE_LOADER UNUSED(cmp_value); @@ -3183,18 +2953,12 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) JUMP_TO_ADDRESS(jump_to_address); } #endif - - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); - #endif - break; } case OP_JUMP: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) TRACE("jump/1 label=%i\n", label); USED_BY_TRACE(label); @@ -3210,20 +2974,14 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) goto schedule_in; } #endif - - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); - #endif - break; } case OP_MOVE: { - int next_off = 1; term src_value; - DECODE_COMPACT_TERM(src_value, code, i, next_off); + DECODE_COMPACT_TERM(src_value, pc); dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("move/2 %lx, %c%i\n", src_value, T_DEST_REG(dreg)); @@ -3235,19 +2993,16 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("move/2\n"); UNUSED(src_value) #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_GET_LIST: { - int next_off = 1; term src_value; - DECODE_COMPACT_TERM(src_value, code, i, next_off) + DECODE_COMPACT_TERM(src_value, pc) dreg_t head_dreg; - DECODE_DEST_REGISTER(head_dreg, code, i, next_off); + DECODE_DEST_REGISTER(head_dreg, pc); dreg_t tail_dreg; - DECODE_DEST_REGISTER(tail_dreg, code, i, next_off); + DECODE_DEST_REGISTER(tail_dreg, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("get_list/3 %lx, %c%i, %c%i\n", src_value, T_DEST_REG(head_dreg), T_DEST_REG(tail_dreg)); @@ -3263,19 +3018,16 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("get_list/2\n"); UNUSED(src_value) #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_GET_TUPLE_ELEMENT: { - int next_off = 1; term src_value; - DECODE_COMPACT_TERM(src_value, code, i, next_off); + DECODE_COMPACT_TERM(src_value, pc); uint32_t element; - DECODE_LITERAL(element, code, i, next_off); + DECODE_LITERAL(element, pc); dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); TRACE("get_tuple_element/2, element=%i, dest=%c%i\n", element, T_DEST_REG(dreg)); USED_BY_TRACE(element); @@ -3292,19 +3044,16 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #ifdef IMPL_CODE_LOADER UNUSED(src_value) #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_SET_TUPLE_ELEMENT: { - int next_off = 1; term new_element; - DECODE_COMPACT_TERM(new_element, code, i, next_off); + DECODE_COMPACT_TERM(new_element, pc); term tuple; - DECODE_COMPACT_TERM(tuple, code, i, next_off); + DECODE_COMPACT_TERM(tuple, pc); uint32_t position; - DECODE_LITERAL(position, code, i, next_off); + DECODE_LITERAL(position, pc); TRACE("set_tuple_element/2\n"); @@ -3321,17 +3070,15 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) UNUSED(position); UNUSED(new_element); #endif - NEXT_INSTRUCTION(next_off); break; } case OP_PUT_LIST: { - int next_off = 1; term head; - DECODE_COMPACT_TERM(head, code, i, next_off); + DECODE_COMPACT_TERM(head, pc); term tail; - DECODE_COMPACT_TERM(tail, code, i, next_off); + DECODE_COMPACT_TERM(tail, pc); #ifdef IMPL_EXECUTE_LOOP // MEMORY_NO_SHRINK because put_list is classified as put in beam_ssa_codegen.erl @@ -3342,7 +3089,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_CODE_LOADER TRACE("put_list/3\n"); @@ -3355,15 +3102,12 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) term t = term_list_init_prepend(list_elem, head, tail); WRITE_REGISTER(dreg, t); #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_PUT_TUPLE: { - int next_off = 1; uint32_t size; - DECODE_LITERAL(size, code, i, next_off); + DECODE_LITERAL(size, pc); #ifdef IMPL_EXECUTE_LOOP // MEMORY_NO_SHRINK because put_tuple is classified as put in beam_ssa_codegen.erl @@ -3374,7 +3118,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); TRACE("put_tuple/2 size=%u, dest=%c%i\n", (unsigned) size, T_DEST_REG(dreg)); USED_BY_TRACE(dreg); @@ -3384,13 +3128,12 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif for (uint32_t j = 0; j < size; j++) { - if (code[i + next_off] != OP_PUT) { - fprintf(stderr, "Expected put, got opcode: %i\n", code[i + next_off]); + if (*pc++ != OP_PUT) { + fprintf(stderr, "Expected put, got opcode: %i\n", pc[-1]); AVM_ABORT(); } - next_off++; term put_value; - DECODE_COMPACT_TERM(put_value, code, i, next_off); + DECODE_COMPACT_TERM(put_value, pc); #ifdef IMPL_CODE_LOADER TRACE("put/2\n"); UNUSED(put_value); @@ -3401,8 +3144,6 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) term_put_tuple_element(t, j, put_value); #endif } - - NEXT_INSTRUCTION(next_off); break; } @@ -3414,9 +3155,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } #endif - int next_off = 1; term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) #ifdef IMPL_CODE_LOADER TRACE("badmatch/1\n"); @@ -3432,11 +3172,6 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) RAISE_ERROR(new_error_tuple); #endif - - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); - #endif - break; } @@ -3449,11 +3184,6 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) RAISE_ERROR(IF_CLAUSE_ATOM); #endif - - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(1); - #endif - break; } @@ -3465,9 +3195,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } #endif - int next_off = 1; term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) #ifdef IMPL_CODE_LOADER TRACE("case_end/1\n"); @@ -3483,28 +3212,23 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) RAISE_ERROR(new_error_tuple); #endif - - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); - #endif - break; } case OP_CALL_FUN: { - int next_off = 1; + #ifdef IMPL_EXECUTE_LOOP + remaining_reductions--; + if (UNLIKELY(!remaining_reductions)) { + SCHEDULE_NEXT(mod, pc - 1); + } + #endif uint32_t args_count; - DECODE_LITERAL(args_count, code, i, next_off) + DECODE_LITERAL(args_count, pc) TRACE("call_fun/1, args_count=%i\n", args_count); USED_BY_TRACE(args_count); #ifdef IMPL_EXECUTE_LOOP - remaining_reductions--; - if (UNLIKELY(!remaining_reductions)) { - SCHEDULE_NEXT(mod, INSTRUCTION_POINTER()); - } - term fun = x_regs[args_count]; if (UNLIKELY(!term_is_function(fun))) { // We can gc as we are raising @@ -3517,30 +3241,23 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) RAISE_ERROR(new_error_tuple); } - CALL_FUN(fun, args_count, next_off) - #endif - - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); + CALL_FUN(fun, args_count) #endif break; } case OP_IS_FUNCTION: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) #ifdef IMPL_EXECUTE_LOOP TRACE("is_function/2, label=%i, arg1=%lx\n", label, arg1); - if (term_is_function(arg1)) { - NEXT_INSTRUCTION(next_off); - } else { - i = POINTER_TO_II(mod->labels[label]); + if (!term_is_function(arg1)) { + pc = mod->labels[label]; } #endif @@ -3548,33 +3265,27 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("is_function/2\n"); UNUSED(label) UNUSED(arg1) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_CALL_EXT_ONLY: { - int next_off = 1; + #ifdef IMPL_EXECUTE_LOOP + remaining_reductions--; + if (UNLIKELY(!remaining_reductions)) { + SCHEDULE_NEXT(mod, pc - 1); + } + #endif uint32_t arity; - DECODE_LITERAL(arity, code, i, next_off); + DECODE_LITERAL(arity, pc); uint32_t index; - DECODE_LITERAL(index, code, i, next_off); + DECODE_LITERAL(index, pc); TRACE("call_ext_only/2, arity=%i, index=%i\n", arity, index); USED_BY_TRACE(arity); USED_BY_TRACE(index); - - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); - #endif - #ifdef IMPL_EXECUTE_LOOP - remaining_reductions--; - if (UNLIKELY(!remaining_reductions)) { - SCHEDULE_NEXT(mod, INSTRUCTION_POINTER()); - } - TRACE_CALL_EXT(ctx, mod, "call_ext_only", index, arity); const struct ExportedFunction *func = module_resolve_function(mod, index, glb); @@ -3640,9 +3351,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } case OP_MAKE_FUN2: { - int next_off = 1; uint32_t fun_index; - DECODE_LITERAL(fun_index, code, i, next_off) + DECODE_LITERAL(fun_index, pc) TRACE("make_fun/2, fun_index=%i\n", fun_index); #ifdef IMPL_EXECUTE_LOOP @@ -3653,17 +3363,14 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) x_regs[0] = f; } #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_TRY: { - int next_off = 1; dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) TRACE("try/2, label=%i, reg=%c%i\n", label, T_DEST_REG(dreg)); @@ -3672,15 +3379,12 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) //TODO: here just write to y registers is enough WRITE_REGISTER(dreg, catch_term); #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_TRY_END: { - int next_off = 1; dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); TRACE("try_end/1, reg=%c%i\n", T_DEST_REG(dreg)); @@ -3688,15 +3392,12 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) //TODO: here just write to y registers is enough WRITE_REGISTER(dreg, term_nil()); #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_TRY_CASE: { - int next_off = 1; dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); TRACE("try_case/1, reg=%c%i\n", T_DEST_REG(dreg)); @@ -3704,8 +3405,6 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) // clears the catch value on stack WRITE_REGISTER(dreg, term_nil()); #endif - - NEXT_INSTRUCTION(next_off); break; } @@ -3717,9 +3416,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } #endif - int next_off = 1; term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) #ifdef IMPL_CODE_LOADER TRACE("try_case_end/1\n"); @@ -3735,30 +3433,29 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) RAISE_ERROR(new_error_tuple); #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_RAISE: { - int next_off = 1; + #ifdef IMPL_EXECUTE_LOOP + const uint8_t *saved_pc = pc - 1; + #endif term stacktrace; - DECODE_COMPACT_TERM(stacktrace, code, i, next_off); + DECODE_COMPACT_TERM(stacktrace, pc); term exc_value; - DECODE_COMPACT_TERM(exc_value, code, i, next_off); + DECODE_COMPACT_TERM(exc_value, pc); #ifdef IMPL_CODE_LOADER TRACE("raise/2\n"); UNUSED(stacktrace); UNUSED(exc_value); - NEXT_INSTRUCTION(next_off); #endif #ifdef IMPL_EXECUTE_LOOP TRACE("raise/2 stacktrace=0x%lx exc_value=0x%lx\n", stacktrace, exc_value); x_regs[0] = stacktrace_exception_class(stacktrace); x_regs[1] = exc_value; - x_regs[2] = stacktrace_create_raw(ctx, mod, i, x_regs[0]); + x_regs[2] = stacktrace_create_raw(ctx, mod, saved_pc - code, x_regs[0]); goto handle_error; #endif @@ -3766,11 +3463,10 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } case OP_CATCH: { - int next_off = 1; dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) TRACE("catch/2, label=%i, reg=%c%i\n", label, T_DEST_REG(dreg)); @@ -3779,15 +3475,12 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) // TODO: here just write to y registers is enough WRITE_REGISTER(dreg, catch_term); #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_CATCH_END: { - int next_off = 1; dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); TRACE("catch_end/1, reg=%c%i\n", T_DEST_REG(dreg)); @@ -3830,22 +3523,20 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } } #endif - NEXT_INSTRUCTION(next_off); break; } case OP_BS_ADD: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) term src1; - DECODE_COMPACT_TERM(src1, code, i, next_off); + DECODE_COMPACT_TERM(src1, pc); term src2; - DECODE_COMPACT_TERM(src2, code, i, next_off); + DECODE_COMPACT_TERM(src2, pc); uint32_t unit; - DECODE_LITERAL(unit, code, i, next_off) + DECODE_LITERAL(unit, pc) dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_add/5\n"); @@ -3861,25 +3552,23 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) WRITE_REGISTER(dreg, term_from_int((src1_val + src2_val) * unit)); #endif - NEXT_INSTRUCTION(next_off); break; } case OP_BS_INIT2: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) term size; - DECODE_COMPACT_TERM(size, code, i, next_off) + DECODE_COMPACT_TERM(size, pc) uint32_t words; UNUSED(words); - DECODE_LITERAL(words, code, i, next_off) + DECODE_LITERAL(words, pc) uint32_t regs; UNUSED(regs); - DECODE_LITERAL(regs, code, i, next_off) + DECODE_LITERAL(regs, pc) term flags; UNUSED(flags); - DECODE_COMPACT_TERM(flags, code, i, next_off) + DECODE_COMPACT_TERM(flags, pc) #ifdef IMPL_CODE_LOADER TRACE("bs_init2/6\n"); @@ -3899,29 +3588,26 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("bs_init2/6, fail=%u size=%li words=%u regs=%u dreg=%c%i\n", (unsigned) fail, size_val, (unsigned) words, (unsigned) regs, T_DEST_REG(dreg)); WRITE_REGISTER(dreg, t); #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_BS_INIT_BITS: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) term size; - DECODE_COMPACT_TERM(size, code, i, next_off) + DECODE_COMPACT_TERM(size, pc) uint32_t words; - DECODE_LITERAL(words, code, i, next_off) + DECODE_LITERAL(words, pc) uint32_t regs; - DECODE_LITERAL(regs, code, i, next_off) + DECODE_LITERAL(regs, pc) uint32_t flags_value; - DECODE_LITERAL(flags_value, code, i, next_off) + DECODE_LITERAL(flags_value, pc) #ifdef IMPL_CODE_LOADER TRACE("bs_init_bits/6\n"); @@ -3949,25 +3635,22 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("bs_init_bits/6, fail=%i size=%li words=%i regs=%i dreg=%c%i\n", fail, size_val, words, regs, T_DEST_REG(dreg)); WRITE_REGISTER(dreg, t); #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_BS_UTF8_SIZE: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) term src; - DECODE_COMPACT_TERM(src, code, i, next_off) + DECODE_COMPACT_TERM(src, pc) dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_utf8_size/3"); #endif @@ -3981,19 +3664,16 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } WRITE_REGISTER(dreg, term_from_int(utf8_size)); #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_BS_PUT_UTF8: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) uint32_t flags; - DECODE_LITERAL(flags, code, i, next_off) + DECODE_LITERAL(flags, pc) term src; - DECODE_COMPACT_TERM(src, code, i, next_off) + DECODE_COMPACT_TERM(src, pc) #ifdef IMPL_CODE_LOADER TRACE("bs_put_utf8/3 flags=%x\n", (int) flags); if (flags != 0) { @@ -4021,26 +3701,23 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } ctx->bs_offset += byte_size * 8; #endif - NEXT_INSTRUCTION(next_off); break; } case OP_BS_GET_UTF8: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); term arg2; - DECODE_COMPACT_TERM(arg2, code, i, next_off); + DECODE_COMPACT_TERM(arg2, pc); term arg3; - DECODE_COMPACT_TERM(arg3, code, i, next_off); + DECODE_COMPACT_TERM(arg3, pc); dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_get_utf8/5\n"); - NEXT_INSTRUCTION(next_off); #endif #ifdef IMPL_EXECUTE_LOOP @@ -4056,11 +3733,10 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) bool is_valid = bitstring_match_utf8(src_bin, (size_t) offset_bits, &val, &out_size); if (!is_valid) { - i = POINTER_TO_II(mod->labels[fail]); + pc = mod->labels[fail]; } else { term_set_match_state_offset(src, offset_bits + (out_size * 8)); WRITE_REGISTER(dreg, term_from_int(val)); - NEXT_INSTRUCTION(next_off); } #endif @@ -4068,19 +3744,17 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } case OP_BS_SKIP_UTF8: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); term arg2; - DECODE_COMPACT_TERM(arg2, code, i, next_off); + DECODE_COMPACT_TERM(arg2, pc); term arg3; - DECODE_COMPACT_TERM(arg3, code, i, next_off); + DECODE_COMPACT_TERM(arg3, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_skip_utf8/4\n"); - NEXT_INSTRUCTION(next_off); #endif #ifdef IMPL_EXECUTE_LOOP @@ -4096,10 +3770,9 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) bool is_valid = bitstring_match_utf8(src_bin, (size_t) offset_bits, &c, &out_size); if (!is_valid) { - i = POINTER_TO_II(mod->labels[fail]); + pc = mod->labels[fail]; } else { term_set_match_state_offset(src, offset_bits + (out_size * 8)); - NEXT_INSTRUCTION(next_off); } #endif @@ -4107,13 +3780,12 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } case OP_BS_UTF16_SIZE: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) term src; - DECODE_COMPACT_TERM(src, code, i, next_off) + DECODE_COMPACT_TERM(src, pc) dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_utf16_size/3"); #endif @@ -4127,19 +3799,16 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } WRITE_REGISTER(dreg, term_from_int(utf16_size)); #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_BS_PUT_UTF16: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) uint32_t flags; - DECODE_LITERAL(flags, code, i, next_off) + DECODE_LITERAL(flags, pc) term src; - DECODE_COMPACT_TERM(src, code, i, next_off) + DECODE_COMPACT_TERM(src, pc) #ifdef IMPL_CODE_LOADER TRACE("bs_put_utf16/3 flags=%x\n", (int) flags); if (flags != 0 && flags != LittleEndianInteger && flags != NativeEndianInteger) { @@ -4167,26 +3836,23 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } ctx->bs_offset += byte_size * 8; #endif - NEXT_INSTRUCTION(next_off); break; } case OP_BS_GET_UTF16: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); term arg2; - DECODE_COMPACT_TERM(arg2, code, i, next_off); + DECODE_COMPACT_TERM(arg2, pc); term flags; - DECODE_LITERAL(flags, code, i, next_off); + DECODE_LITERAL(flags, pc); dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_get_utf16/5\n"); - NEXT_INSTRUCTION(next_off); #endif #ifdef IMPL_EXECUTE_LOOP @@ -4202,11 +3868,10 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) bool is_valid = bitstring_match_utf16(src_bin, (size_t) offset_bits, &val, &out_size, flags); if (!is_valid) { - i = POINTER_TO_II(mod->labels[fail]); + pc = mod->labels[fail]; } else { term_set_match_state_offset(src, offset_bits + (out_size * 8)); WRITE_REGISTER(dreg, term_from_int(val)); - NEXT_INSTRUCTION(next_off); } #endif @@ -4214,19 +3879,17 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } case OP_BS_SKIP_UTF16: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); term arg2; - DECODE_COMPACT_TERM(arg2, code, i, next_off); + DECODE_COMPACT_TERM(arg2, pc); term flags; - DECODE_LITERAL(flags, code, i, next_off); + DECODE_LITERAL(flags, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_skip_utf16/5\n"); - NEXT_INSTRUCTION(next_off); #endif #ifdef IMPL_EXECUTE_LOOP @@ -4242,10 +3905,9 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) bool is_valid = bitstring_match_utf16(src_bin, (size_t) offset_bits, &val, &out_size, flags); if (!is_valid) { - i = POINTER_TO_II(mod->labels[fail]); + pc = mod->labels[fail]; } else { term_set_match_state_offset(src, offset_bits + (out_size * 8)); - NEXT_INSTRUCTION(next_off); } #endif @@ -4253,13 +3915,12 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } case OP_BS_PUT_UTF32: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) uint32_t flags; - DECODE_LITERAL(flags, code, i, next_off) + DECODE_LITERAL(flags, pc) term src; - DECODE_COMPACT_TERM(src, code, i, next_off) + DECODE_COMPACT_TERM(src, pc) #ifdef IMPL_CODE_LOADER TRACE("bs_put_utf32/3 flags=%x\n", (int) flags); if (flags != 0 && flags != LittleEndianInteger && flags != NativeEndianInteger) { @@ -4286,26 +3947,23 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } ctx->bs_offset += 4 * 8; #endif - NEXT_INSTRUCTION(next_off); break; } case OP_BS_GET_UTF32: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); term arg2; - DECODE_COMPACT_TERM(arg2, code, i, next_off); + DECODE_COMPACT_TERM(arg2, pc); term flags; - DECODE_LITERAL(flags, code, i, next_off); + DECODE_LITERAL(flags, pc); dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_get_utf32/5\n"); - NEXT_INSTRUCTION(next_off); #endif #ifdef IMPL_EXECUTE_LOOP @@ -4320,11 +3978,10 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) bool is_valid = bitstring_match_utf32(src_bin, (size_t) offset_bits, &val, flags); if (!is_valid) { - i = POINTER_TO_II(mod->labels[fail]); + pc = mod->labels[fail]; } else { term_set_match_state_offset(src, offset_bits + 32); WRITE_REGISTER(dreg, term_from_int(val)); - NEXT_INSTRUCTION(next_off); } #endif @@ -4332,19 +3989,17 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } case OP_BS_SKIP_UTF32: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); term arg2; - DECODE_COMPACT_TERM(arg2, code, i, next_off); + DECODE_COMPACT_TERM(arg2, pc); term flags; - DECODE_LITERAL(flags, code, i, next_off); + DECODE_LITERAL(flags, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_skip_utf32/5\n"); - NEXT_INSTRUCTION(next_off); #endif #ifdef IMPL_EXECUTE_LOOP @@ -4359,10 +4014,9 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) bool is_valid = bitstring_match_utf32(src_bin, (size_t) offset_bits, &val, flags); if (!is_valid) { - i = POINTER_TO_II(mod->labels[fail]); + pc = mod->labels[fail]; } else { term_set_match_state_offset(src, offset_bits + 32); - NEXT_INSTRUCTION(next_off); } #endif @@ -4370,7 +4024,6 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } case OP_BS_INIT_WRITABLE: { - int next_off = 1; TRACE("bs_init_writable/0\n"); @@ -4384,33 +4037,30 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) ctx->bs_offset = 0; x_regs[0] = t; #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_BS_APPEND: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) term size; - DECODE_COMPACT_TERM(size, code, i, next_off) + DECODE_COMPACT_TERM(size, pc) term extra; UNUSED(extra); - DECODE_COMPACT_TERM(extra, code, i, next_off) + DECODE_COMPACT_TERM(extra, pc) term live; UNUSED(live); - DECODE_COMPACT_TERM(live, code, i, next_off) + DECODE_COMPACT_TERM(live, pc) uint32_t unit; - DECODE_LITERAL(unit, code, i, next_off); + DECODE_LITERAL(unit, pc); term src; #ifdef IMPL_EXECUTE_LOOP - int src_off = next_off; + const uint8_t *src_pc = pc; #endif - DECODE_COMPACT_TERM(src, code, i, next_off) + DECODE_COMPACT_TERM(src, pc) term flags; UNUSED(flags); - DECODE_COMPACT_TERM(flags, code, i, next_off) + DECODE_COMPACT_TERM(flags, pc) #ifdef IMPL_CODE_LOADER TRACE("bs_append/8\n"); @@ -4440,11 +4090,11 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("bs_append/8, fail=%u size=%li unit=%u src=0x%lx dreg=%c%i\n", (unsigned) fail, size_val, (unsigned) unit, src, T_DEST_REG(dreg)); - DECODE_COMPACT_TERM(src, code, i, src_off) + DECODE_COMPACT_TERM(src, src_pc) term t = term_create_empty_binary(src_size + size_val / 8, &ctx->heap, ctx->global); memcpy((void *) term_binary_data(t), (void *) term_binary_data(src), src_size); @@ -4453,27 +4103,24 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) WRITE_REGISTER(dreg, t); #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_BS_PRIVATE_APPEND: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) term size; - DECODE_COMPACT_TERM(size, code, i, next_off) + DECODE_COMPACT_TERM(size, pc) uint32_t unit; - DECODE_LITERAL(unit, code, i, next_off); + DECODE_LITERAL(unit, pc); term src; #ifdef IMPL_EXECUTE_LOOP - int src_off = next_off; + const uint8_t *src_pc = pc; #endif - DECODE_COMPACT_TERM(src, code, i, next_off) + DECODE_COMPACT_TERM(src, pc) term flags; UNUSED(flags); - DECODE_COMPACT_TERM(flags, code, i, next_off) + DECODE_COMPACT_TERM(flags, pc) #ifdef IMPL_CODE_LOADER TRACE("bs_private_append/6\n"); @@ -4494,7 +4141,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) if (UNLIKELY(memory_ensure_free_opt(ctx, src_size + term_binary_heap_size(size_val / 8), MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } - DECODE_COMPACT_TERM(src, code, i, src_off) + DECODE_COMPACT_TERM(src, src_pc) term t = term_create_empty_binary(src_size + size_val / 8, &ctx->heap, ctx->global); memcpy((void *) term_binary_data(t), (void *) term_binary_data(src), src_size); @@ -4503,29 +4150,26 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("bs_private_append/6, fail=%u size=%li unit=%u src=0x%lx dreg=%c%i\n", (unsigned) fail, size_val, (unsigned) unit, src, T_DEST_REG(dreg)); WRITE_REGISTER(dreg, t); #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_BS_PUT_INTEGER: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) term size; - DECODE_COMPACT_TERM(size, code, i, next_off) + DECODE_COMPACT_TERM(size, pc) uint32_t unit; - DECODE_LITERAL(unit, code, i, next_off); + DECODE_LITERAL(unit, pc); uint32_t flags_value; - DECODE_LITERAL(flags_value, code, i, next_off) + DECODE_LITERAL(flags_value, pc) term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_put_integer/5\n"); @@ -4548,22 +4192,20 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) ctx->bs_offset += size_value * unit; #endif - NEXT_INSTRUCTION(next_off); break; } case OP_BS_PUT_BINARY: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) term size; - DECODE_COMPACT_TERM(size, code, i, next_off) + DECODE_COMPACT_TERM(size, pc) uint32_t unit; - DECODE_LITERAL(unit, code, i, next_off); + DECODE_LITERAL(unit, pc); uint32_t flags_value; - DECODE_LITERAL(flags_value, code, i, next_off) + DECODE_LITERAL(flags_value, pc) term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_put_binary/5\n"); @@ -4608,16 +4250,14 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } ctx->bs_offset += 8 * size_val; #endif - NEXT_INSTRUCTION(next_off); break; } case OP_BS_PUT_STRING: { - int next_off = 1; uint32_t size; - DECODE_LITERAL(size, code, i, next_off); + DECODE_LITERAL(size, pc); uint32_t offset; - DECODE_LITERAL(offset, code, i, next_off); + DECODE_LITERAL(offset, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_put_string/2\n"); @@ -4645,23 +4285,21 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) memcpy((char *) term_binary_data(ctx->bs) + ctx->bs_offset / 8, str, size); ctx->bs_offset += 8 * size; #endif - NEXT_INSTRUCTION(next_off); break; } case OP_BS_START_MATCH2: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) #ifdef IMPL_EXECUTE_LOOP - int next_off_back = next_off; + const uint8_t *src_pc = pc; #endif term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); term arg2; - DECODE_COMPACT_TERM(arg2, code, i, next_off); + DECODE_COMPACT_TERM(arg2, pc); term slots_term; - DECODE_COMPACT_TERM(slots_term, code, i, next_off); + DECODE_COMPACT_TERM(slots_term, pc); #ifdef IMPL_EXECUTE_LOOP int slots = term_to_int(slots_term); @@ -4670,11 +4308,11 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) RAISE_ERROR(OUT_OF_MEMORY_ATOM); } - DECODE_COMPACT_TERM(src, code, i, next_off_back); + DECODE_COMPACT_TERM(src, src_pc); #endif dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_start_match2/5\n"); @@ -4684,18 +4322,13 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("bs_start_match2/5, fail=%i src=0x%lx arg2=0x%lx arg3=0x%lx dreg=%c%i\n", fail, src, arg2, slots_term, T_DEST_REG(dreg)); if (!(term_is_binary(src) || term_is_match_state(src))) { WRITE_REGISTER(dreg, src); - i = POINTER_TO_II(mod->labels[fail]); + pc = mod->labels[fail]; } else { term match_state = term_alloc_bin_match_state(src, slots, &ctx->heap); WRITE_REGISTER(dreg, match_state); - NEXT_INSTRUCTION(next_off); } #endif - - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); - #endif break; } @@ -4707,15 +4340,14 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } #endif - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); term live; - DECODE_COMPACT_TERM(live, code, i, next_off); + DECODE_COMPACT_TERM(live, pc); dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_start_match3/4\n"); @@ -4725,29 +4357,23 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("bs_start_match3/4, fail=%i src=0x%lx live=0x%lx dreg=%c%i\n", fail, src, live, T_DEST_REG(dreg)); if (!(term_is_binary(src) || term_is_match_state(src))) { WRITE_REGISTER(dreg, src); - i = POINTER_TO_II(mod->labels[fail]); + pc = mod->labels[fail]; } else { term match_state = term_alloc_bin_match_state(src, 0, &ctx->heap); WRITE_REGISTER(dreg, match_state); - NEXT_INSTRUCTION(next_off); } #endif - - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); - #endif break; } case OP_BS_GET_POSITION: { - int next_off = 1; term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); term live; - DECODE_COMPACT_TERM(live, code, i, next_off); + DECODE_COMPACT_TERM(live, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_get_position/3\n"); @@ -4763,22 +4389,19 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) WRITE_REGISTER(dreg, offset_term); #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_BS_GET_TAIL: { - int next_off = 1; term src; #ifdef IMPL_EXECUTE_LOOP - int src_off = next_off; + const uint8_t *src_pc = pc; #endif - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); dreg_unsafe_t dreg; - DECODE_DEST_REGISTER_UNSAFE(dreg, code, i, next_off); + DECODE_DEST_REGISTER_UNSAFE(dreg, pc); term live; - DECODE_COMPACT_TERM(live, code, i, next_off); + DECODE_COMPACT_TERM(live, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_get_tail/3\n"); @@ -4809,7 +4432,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) if (UNLIKELY(memory_ensure_free_opt(ctx, heap_size, MEMORY_CAN_SHRINK) != MEMORY_GC_OK)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } - DECODE_COMPACT_TERM(src, code, i, src_off); + DECODE_COMPACT_TERM(src, src_pc); bs_bin = term_get_match_state_binary(src); term t = term_maybe_create_sub_binary(bs_bin, start_pos, new_bin_size, &ctx->heap, ctx->global); WRITE_REGISTER_UNSAFE(dreg, t); @@ -4817,17 +4440,14 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } } #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_BS_SET_POSITION: { - int next_off = 1; term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); term pos; - DECODE_COMPACT_TERM(pos, code, i, next_off); + DECODE_COMPACT_TERM(pos, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_set_position/2\n"); @@ -4841,21 +4461,18 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("bs_set_position/2 src=0x%lx pos=%li\n", src, pos_val); term_set_match_state_offset(src, pos_val); #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_BS_MATCH_STRING: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); uint32_t bits; - DECODE_LITERAL(bits, code, i, next_off); + DECODE_LITERAL(bits, pc); uint32_t offset; - DECODE_LITERAL(offset, code, i, next_off); + DECODE_LITERAL(offset, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_match_string/5\n"); @@ -4891,22 +4508,16 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) JUMP_TO_ADDRESS(mod->labels[fail]); } else { term_set_match_state_offset(src, bs_offset + bits); - NEXT_INSTRUCTION(next_off); } #endif - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); - #endif - break; } case OP_BS_SAVE2: { - int next_off = 1; term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); term index = 0; - DECODE_COMPACT_TERM(index, code, i, next_off); + DECODE_COMPACT_TERM(index, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_save2/2\n"); @@ -4928,17 +4539,14 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("bs_save2/2, src=0x%lx pos=%li\n", src, index == START_ATOM ? -1 : index_val); #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_BS_RESTORE2: { - int next_off = 1; term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); term index = 0; - DECODE_COMPACT_TERM(index, code, i, next_off); + DECODE_COMPACT_TERM(index, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_restore2/5\n"); @@ -4959,23 +4567,20 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("bs_restore2/2, src=0x%lx pos=%li\n", src, index == START_ATOM ? -1 : index_val); #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_BS_SKIP_BITS2: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); term size; - DECODE_COMPACT_TERM(size, code, i, next_off); + DECODE_COMPACT_TERM(size, pc); uint32_t unit; - DECODE_LITERAL(unit, code, i, next_off); + DECODE_LITERAL(unit, pc); uint32_t flags_value; - DECODE_LITERAL(flags_value, code, i, next_off) + DECODE_LITERAL(flags_value, pc) #ifdef IMPL_CODE_LOADER TRACE("bs_skip_bits2/5\n"); @@ -5000,24 +4605,19 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) JUMP_TO_ADDRESS(mod->labels[fail]); } else { term_set_match_state_offset(src, bs_offset + increment); - NEXT_INSTRUCTION(next_off); } #endif - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); - #endif break; } case OP_BS_TEST_UNIT: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); uint32_t unit; - DECODE_LITERAL(unit, code, i, next_off); + DECODE_LITERAL(unit, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_test_unit/3\n"); @@ -5032,24 +4632,18 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) if ((term_binary_size(src) * 8 - bs_offset) % unit != 0) { TRACE("bs_test_unit: Available bits in source not evenly divisible by unit"); JUMP_TO_ADDRESS(mod->labels[fail]); - } else { - NEXT_INSTRUCTION(next_off); } #endif - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); - #endif break; } case OP_BS_TEST_TAIL2: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); uint32_t bits; - DECODE_LITERAL(bits, code, i, next_off); + DECODE_LITERAL(bits, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_test_tail2/3\n"); @@ -5066,30 +4660,24 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) if ((term_binary_size(bs_bin) * 8 - bs_offset) != (unsigned int) bits) { TRACE("bs_test_tail2: Expected exactly %u bits remaining, but remaining=%u\n", (unsigned) bits, (unsigned) (term_binary_size(bs_bin) * 8 - bs_offset)); JUMP_TO_ADDRESS(mod->labels[fail]); - } else { - NEXT_INSTRUCTION(next_off); } #endif - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); - #endif break; } case OP_BS_GET_INTEGER2: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); term arg2; - DECODE_COMPACT_TERM(arg2, code, i, next_off); + DECODE_COMPACT_TERM(arg2, pc); term size; - DECODE_COMPACT_TERM(size, code, i, next_off); + DECODE_COMPACT_TERM(size, pc); uint32_t unit; - DECODE_LITERAL(unit, code, i, next_off); + DECODE_LITERAL(unit, pc); uint32_t flags_value; - DECODE_LITERAL(flags_value, code, i, next_off) + DECODE_LITERAL(flags_value, pc) #ifdef IMPL_CODE_LOADER TRACE("bs_get_integer2/7\n"); @@ -5121,36 +4709,31 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_EXECUTE_LOOP WRITE_REGISTER(dreg, t); - NEXT_INSTRUCTION(next_off); } #endif - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); - #endif break; } case OP_BS_GET_BINARY2: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off) + DECODE_LABEL(fail, pc) term src; #ifdef IMPL_EXECUTE_LOOP - int src_offset = next_off; + const uint8_t *src_pc = pc; #endif - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); term arg2; - DECODE_COMPACT_TERM(arg2, code, i, next_off); + DECODE_COMPACT_TERM(arg2, pc); term size; - DECODE_COMPACT_TERM(size, code, i, next_off); + DECODE_COMPACT_TERM(size, pc); uint32_t unit; - DECODE_LITERAL(unit, code, i, next_off); + DECODE_LITERAL(unit, pc); uint32_t flags_value; - DECODE_LITERAL(flags_value, code, i, next_off) + DECODE_LITERAL(flags_value, pc) #ifdef IMPL_CODE_LOADER TRACE("bs_get_binary2/7\n"); @@ -5199,31 +4782,25 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_EXECUTE_LOOP // re-compute src - DECODE_COMPACT_TERM(src, code, i, src_offset); + DECODE_COMPACT_TERM(src, src_pc); bs_bin = term_get_match_state_binary(src); term t = term_maybe_create_sub_binary(bs_bin, bs_offset / unit, size_val, &ctx->heap, ctx->global); WRITE_REGISTER(dreg, t); - NEXT_INSTRUCTION(next_off); } #endif - - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); - #endif break; } case OP_BS_CONTEXT_TO_BINARY: { // Do not check if dreg is a binary or not // In case it is not a binary or a match state, dreg will not be changed. - int next_off = 1; dreg_unsafe_t dreg; - DECODE_DEST_REGISTER_UNSAFE(dreg, code, i, next_off); + DECODE_DEST_REGISTER_UNSAFE(dreg, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_context_to_binary/1\n"); @@ -5254,30 +4831,26 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } WRITE_REGISTER_UNSAFE(dreg, bin); #endif - - - NEXT_INSTRUCTION(next_off); break; } case OP_APPLY: { - int next_off = 1; + #ifdef IMPL_EXECUTE_LOOP + // save pc in case of error + const uint8_t *orig_pc = pc - 1; + + remaining_reductions--; + if (UNLIKELY(!remaining_reductions)) { + SCHEDULE_NEXT(mod, orig_pc); + } + #endif uint32_t arity; - DECODE_LITERAL(arity, code, i, next_off) + DECODE_LITERAL(arity, pc) #ifdef IMPL_EXECUTE_LOOP term module = x_regs[arity]; term function = x_regs[arity + 1]; TRACE("apply/1, module=%lu, function=%lu arity=%i\n", module, function, arity); - remaining_reductions--; - if (UNLIKELY(!remaining_reductions)) { - SCHEDULE_NEXT(mod, INSTRUCTION_POINTER()); - } - - // save instruction offset in case of error - int orig_i = i; - NEXT_INSTRUCTION(next_off); - if (UNLIKELY(!term_is_atom(module) || !term_is_atom(function))) { RAISE_ERROR(BADARG_ATOM); } @@ -5289,47 +4862,46 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) term native_return; if (maybe_call_native(ctx, module_name, function_name, arity, &native_return)) { - PROCESS_MAYBE_TRAP_RETURN_VALUE_RESTORE_I(native_return, orig_i); + PROCESS_MAYBE_TRAP_RETURN_VALUE_RESTORE_PC(native_return, orig_pc); x_regs[0] = native_return; } else { Module *target_module = globalcontext_get_module(ctx->global, module_name); if (IS_NULL_PTR(target_module)) { - i = orig_i; + pc = orig_pc; HANDLE_ERROR(); } int target_label = module_search_exported_function(target_module, function_name, arity, glb); if (target_label == 0) { - i = orig_i; + pc = orig_pc; HANDLE_ERROR(); } - ctx->cp = module_address(mod->module_index, i); + ctx->cp = module_address(mod->module_index, pc - code); JUMP_TO_LABEL(target_module, target_label); } #endif #ifdef IMPL_CODE_LOADER TRACE("apply/1 arity=%i\n", arity); - NEXT_INSTRUCTION(next_off); #endif break; } case OP_APPLY_LAST: { - int next_off = 1; + #ifdef IMPL_EXECUTE_LOOP + remaining_reductions--; + if (UNLIKELY(!remaining_reductions)) { + SCHEDULE_NEXT(mod, pc - 1); + } + #endif uint32_t arity; - DECODE_LITERAL(arity, code, i, next_off) + DECODE_LITERAL(arity, pc) uint32_t n_words; - DECODE_LITERAL(n_words, code, i, next_off); + DECODE_LITERAL(n_words, pc); #ifdef IMPL_EXECUTE_LOOP term module = x_regs[arity]; term function = x_regs[arity + 1]; TRACE("apply_last/1, module=%lu, function=%lu arity=%i deallocate=%i\n", module, function, arity, n_words); - remaining_reductions--; - if (UNLIKELY(!remaining_reductions)) { - SCHEDULE_NEXT(mod, INSTRUCTION_POINTER()); - } - ctx->cp = ctx->e[n_words]; ctx->e += (n_words + 1); @@ -5362,25 +4934,21 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif #ifdef IMPL_CODE_LOADER TRACE("apply_last/1 arity=%i deallocate=%i\n", arity, n_words); - NEXT_INSTRUCTION(next_off); #endif break; } case OP_IS_BOOLEAN: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) #ifdef IMPL_EXECUTE_LOOP TRACE("is_boolean/2, label=%i, arg1=%lx\n", label, arg1); - if ((arg1 == TRUE_ATOM) || (arg1 == FALSE_ATOM)) { - NEXT_INSTRUCTION(next_off); - } else { - i = POINTER_TO_II(mod->labels[label]); + if ((arg1 != TRUE_ATOM) && (arg1 != FALSE_ATOM)) { + pc = mod->labels[label]; } #endif @@ -5388,20 +4956,18 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("is_boolean/2\n"); UNUSED(label) UNUSED(arg1) - NEXT_INSTRUCTION(next_off); - #endif + #endif break; } case OP_IS_FUNCTION2: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) unsigned int arity; - DECODE_INTEGER(arity, code, i, next_off) + DECODE_INTEGER(arity, pc) #ifdef IMPL_EXECUTE_LOOP TRACE("is_function2/3, label=%i, arg1=%lx, arity=%i\n", label, arg1, arity); @@ -5428,13 +4994,11 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) fun_arity = fun_arity_and_freeze - fun_n_freeze; } - if (arity == fun_arity) { - NEXT_INSTRUCTION(next_off); - } else { - i = POINTER_TO_II(mod->labels[label]); + if (arity != fun_arity) { + pc = mod->labels[label]; } } else { - i = POINTER_TO_II(mod->labels[label]); + pc = mod->labels[label]; } #endif @@ -5442,22 +5006,20 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("is_function/3\n"); UNUSED(label) UNUSED(arg1) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_GC_BIF1: { - int next_off = 1; uint32_t f_label; - DECODE_LABEL(f_label, code, i, next_off); + DECODE_LABEL(f_label, pc); uint32_t live; - DECODE_LITERAL(live, code, i, next_off); + DECODE_LITERAL(live, pc); uint32_t bif; - DECODE_LITERAL(bif, code, i, next_off); //s? + DECODE_LITERAL(bif, pc); //s? term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) #ifdef IMPL_EXECUTE_LOOP const struct ExportedFunction *exported_bif = mod->imported_funcs[bif]; @@ -5469,7 +5031,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("gc_bif1/5 fail_lbl=%i, live=%i, bif=%i, arg1=0x%lx, dest=%c%i\n", f_label, live, bif, arg1, T_DEST_REG(dreg)); @@ -5487,23 +5049,20 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif UNUSED(f_label) - - NEXT_INSTRUCTION(next_off); break; } case OP_GC_BIF2: { - int next_off = 1; uint32_t f_label; - DECODE_LABEL(f_label, code, i, next_off); + DECODE_LABEL(f_label, pc); uint32_t live; - DECODE_LITERAL(live, code, i, next_off); + DECODE_LITERAL(live, pc); uint32_t bif; - DECODE_LITERAL(bif, code, i, next_off); //s? + DECODE_LITERAL(bif, pc); //s? term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off); + DECODE_COMPACT_TERM(arg1, pc); term arg2; - DECODE_COMPACT_TERM(arg2, code, i, next_off); + DECODE_COMPACT_TERM(arg2, pc); #ifdef IMPL_EXECUTE_LOOP const struct ExportedFunction *exported_bif = mod->imported_funcs[bif]; @@ -5515,7 +5074,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("gc_bif2/6 fail_lbl=%i, live=%i, bif=%i, arg1=0x%lx, arg2=0x%lx, dest=%c%i\n", f_label, live, bif, arg1, arg2, T_DEST_REG(dreg)); @@ -5534,52 +5093,43 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif UNUSED(f_label) - - NEXT_INSTRUCTION(next_off); break; } //TODO: stub, always false case OP_IS_BITSTR: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) #ifdef IMPL_EXECUTE_LOOP TRACE("is_bitstr/2, label=%i, arg1=%lx\n", label, arg1); - if (0) { - NEXT_INSTRUCTION(next_off); - } else { - i = POINTER_TO_II(mod->labels[label]); - } + pc = mod->labels[label]; #endif #ifdef IMPL_CODE_LOADER TRACE("is_bitstr/2\n"); UNUSED(arg1) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_GC_BIF3: { - int next_off = 1; uint32_t f_label; - DECODE_LABEL(f_label, code, i, next_off); + DECODE_LABEL(f_label, pc); uint32_t live; - DECODE_LITERAL(live, code, i, next_off); + DECODE_LITERAL(live, pc); uint32_t bif; - DECODE_LITERAL(bif, code, i, next_off); //s? + DECODE_LITERAL(bif, pc); //s? term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off); + DECODE_COMPACT_TERM(arg1, pc); term arg2; - DECODE_COMPACT_TERM(arg2, code, i, next_off); + DECODE_COMPACT_TERM(arg2, pc); term arg3; - DECODE_COMPACT_TERM(arg3, code, i, next_off); + DECODE_COMPACT_TERM(arg3, pc); #ifdef IMPL_EXECUTE_LOOP const struct ExportedFunction *exported_bif = mod->imported_funcs[bif]; @@ -5591,7 +5141,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("gc_bif3/7 fail_lbl=%i, live=%i, bif=%i, arg1=0x%lx, arg2=0x%lx, arg3=0x%lx, dest=%c%i\n", f_label, live, bif, arg1, arg2, arg3, T_DEST_REG(dreg)); @@ -5611,17 +5161,14 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif UNUSED(f_label) - - NEXT_INSTRUCTION(next_off); break; } case OP_TRIM: { - int next_off = 1; uint32_t n_words; - DECODE_LITERAL(n_words, code, i, next_off); + DECODE_LITERAL(n_words, pc); uint32_t n_remaining; - DECODE_LITERAL(n_remaining, code, i, next_off); + DECODE_LITERAL(n_remaining, pc); TRACE("trim/2 words=%i, remaining=%i\n", n_words, n_remaining); USED_BY_TRACE(n_words); @@ -5634,75 +5181,66 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif UNUSED(n_remaining) - - NEXT_INSTRUCTION(next_off); break; } //TODO: stub, implement recv_mark/1 //it looks like it can be safely left unimplemented case OP_RECV_MARK: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off); + DECODE_LABEL(label, pc); TRACE("recv_mark/1 label=%i\n", label); USED_BY_TRACE(label); - - NEXT_INSTRUCTION(next_off); break; } //TODO: stub, implement recv_set/1 //it looks like it can be safely left unimplemented case OP_RECV_SET: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off); + DECODE_LABEL(label, pc); TRACE("recv_set/1 label=%i\n", label); USED_BY_TRACE(label); - - NEXT_INSTRUCTION(next_off); break; } case OP_LINE: { - int next_off = 1; + #ifdef IMPL_CODE_LOADER + const uint8_t *saved_pc = pc -1; + #endif uint32_t line_number; - DECODE_LITERAL(line_number, code, i, next_off); + DECODE_LITERAL(line_number, pc); TRACE("line/1: %i\n", line_number); #ifdef IMPL_CODE_LOADER - module_insert_line_ref_offset(mod, line_number, i); + module_insert_line_ref_offset(mod, line_number, saved_pc - code); #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_PUT_MAP_ASSOC: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term src; #ifdef IMPL_EXECUTE_LOOP - int src_offset = next_off; + const uint8_t *src_pc = pc; #endif - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); dreg_unsafe_t dreg; - DECODE_DEST_REGISTER_UNSAFE(dreg, code, i, next_off); + DECODE_DEST_REGISTER_UNSAFE(dreg, pc); uint32_t live; - DECODE_LITERAL(live, code, i, next_off); + DECODE_LITERAL(live, pc); TRACE("put_map_assoc/5: label: %i src: 0x%lx dest=%c%i live: %i\n", label, src, T_DEST_REG_UNSAFE(dreg), live); - DECODE_EXTENDED_LIST_TAG(code, i, next_off); + DECODE_EXTENDED_LIST_TAG(pc); uint32_t list_len; - DECODE_LITERAL(list_len, code, i, next_off); + DECODE_LITERAL(list_len, pc); #ifdef IMPL_EXECUTE_LOOP - int list_off = next_off; + const uint8_t *list_pc = pc; #endif uint32_t num_elements = list_len / 2; // @@ -5713,8 +5251,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif for (uint32_t j = 0; j < num_elements; ++j) { term key, value; - DECODE_COMPACT_TERM(key, code, i, next_off); - DECODE_COMPACT_TERM(value, code, i, next_off); + DECODE_COMPACT_TERM(key, pc); + DECODE_COMPACT_TERM(value, pc); #ifdef IMPL_EXECUTE_LOOP int map_pos = term_find_map_pos(src, key, ctx->global); @@ -5738,7 +5276,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) if (memory_ensure_free_opt(ctx, heap_needed, MEMORY_CAN_SHRINK) != MEMORY_GC_OK) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } - DECODE_COMPACT_TERM(src, code, i, src_offset); + DECODE_COMPACT_TERM(src, src_pc); // // // @@ -5748,8 +5286,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } for (uint32_t j = 0; j < num_elements; j++) { term key, value; - DECODE_COMPACT_TERM(key, code, i, list_off); - DECODE_COMPACT_TERM(value, code, i, list_off); + DECODE_COMPACT_TERM(key, list_pc); + DECODE_COMPACT_TERM(value, list_pc); kv[j].key = key; kv[j].value = value; } @@ -5811,32 +5349,28 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) WRITE_REGISTER_UNSAFE(dreg, map); #endif - - NEXT_INSTRUCTION(next_off); - break; } case OP_PUT_MAP_EXACT: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term src; #ifdef IMPL_EXECUTE_LOOP - int src_offset = next_off; + const uint8_t *src_pc = pc; #endif - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); dreg_unsafe_t dreg; - DECODE_DEST_REGISTER_UNSAFE(dreg, code, i, next_off); + DECODE_DEST_REGISTER_UNSAFE(dreg, pc); uint32_t live; - DECODE_LITERAL(live, code, i, next_off); + DECODE_LITERAL(live, pc); TRACE("put_map_exact/5: label: %i src: 0x%lx dest=%c%i live: %i\n", label, src, T_DEST_REG_UNSAFE(dreg), live); - DECODE_EXTENDED_LIST_TAG(code, i, next_off); + DECODE_EXTENDED_LIST_TAG(pc); uint32_t list_len; - DECODE_LITERAL(list_len, code, i, next_off); + DECODE_LITERAL(list_len, pc); #ifdef IMPL_EXECUTE_LOOP - int list_off = next_off; + const uint8_t *list_pc = pc; #endif uint32_t num_elements = list_len / 2; // @@ -5844,8 +5378,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) // for (uint32_t j = 0; j < num_elements; ++j) { term key, value; - DECODE_COMPACT_TERM(key, code, i, next_off); - DECODE_COMPACT_TERM(value, code, i, next_off); + DECODE_COMPACT_TERM(key, pc); + DECODE_COMPACT_TERM(value, pc); #ifdef IMPL_EXECUTE_LOOP int map_pos = term_find_map_pos(src, key, ctx->global); @@ -5866,7 +5400,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) if (memory_ensure_free_opt(ctx, term_map_size_in_terms_maybe_shared(src_size, true), MEMORY_CAN_SHRINK) != MEMORY_GC_OK) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } - DECODE_COMPACT_TERM(src, code, i, src_offset); + DECODE_COMPACT_TERM(src, src_pc); // // Create a new map of the same size as src and populate with entries from src // @@ -5879,8 +5413,8 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) // for (uint32_t j = 0; j < num_elements; ++j) { term key, value; - DECODE_COMPACT_TERM(key, code, i, list_off); - DECODE_COMPACT_TERM(value, code, i, list_off); + DECODE_COMPACT_TERM(key, list_pc); + DECODE_COMPACT_TERM(value, list_pc); int pos = term_find_map_pos(src, key, ctx->global); if (UNLIKELY(pos == TERM_MAP_MEMORY_ALLOC_FAIL)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); @@ -5889,26 +5423,20 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } WRITE_REGISTER_UNSAFE(dreg, map); #endif - - NEXT_INSTRUCTION(next_off); - break; } case OP_IS_MAP: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) #ifdef IMPL_EXECUTE_LOOP TRACE("is_map/2, label=%i, arg1=%lx\n", label, arg1); - if (term_is_map(arg1)) { - NEXT_INSTRUCTION(next_off); - } else { - i = POINTER_TO_II(mod->labels[label]); + if (!term_is_map(arg1)) { + pc = mod->labels[label]; } #endif @@ -5916,18 +5444,16 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("is_map/2\n"); UNUSED(label) UNUSED(arg1) - NEXT_INSTRUCTION(next_off); #endif break; } case OP_HAS_MAP_FIELDS: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("has_map_fields/3: label: %i src: 0x%lx\n", label, src); @@ -5935,58 +5461,52 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("has_map_fields/3: label: %i\n", label); #endif - DECODE_EXTENDED_LIST_TAG(code, i, next_off); + DECODE_EXTENDED_LIST_TAG(pc); uint32_t list_len; - DECODE_LITERAL(list_len, code, i, next_off); - int fail = 0; - for (uint32_t j = 0; j < list_len && !fail; ++j) { + DECODE_LITERAL(list_len, pc); + for (uint32_t j = 0; j < list_len; ++j) { term key; - DECODE_COMPACT_TERM(key, code, i, next_off); + DECODE_COMPACT_TERM(key, pc); #ifdef IMPL_EXECUTE_LOOP int pos = term_find_map_pos(src, key, ctx->global); if (pos == TERM_MAP_NOT_FOUND) { - i = POINTER_TO_II(mod->labels[label]); - fail = 1; + pc = mod->labels[label]; + break; } else if (pos == TERM_MAP_MEMORY_ALLOC_FAIL) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } #endif } - if (!fail) { - NEXT_INSTRUCTION(next_off); - } break; } case OP_GET_MAP_ELEMENTS: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("get_map_elements/3: label: %i src: 0x%lx\n", label, src); #else TRACE("get_map_elements/3: label: %i\n", label); #endif - DECODE_EXTENDED_LIST_TAG(code, i, next_off); + DECODE_EXTENDED_LIST_TAG(pc); uint32_t list_len; - DECODE_LITERAL(list_len, code, i, next_off); + DECODE_LITERAL(list_len, pc); uint32_t num_elements = list_len / 2; - uint32_t fail = 0; - for (uint32_t j = 0; j < num_elements && !fail; ++j) { + for (uint32_t j = 0; j < num_elements; ++j) { term key; - DECODE_COMPACT_TERM(key, code, i, next_off); + DECODE_COMPACT_TERM(key, pc); dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_EXECUTE_LOOP int pos = term_find_map_pos(src, key, ctx->global); if (pos == TERM_MAP_NOT_FOUND) { - i = POINTER_TO_II(mod->labels[label]); - fail = 1; + pc = mod->labels[label]; + break; } else if (UNLIKELY(pos == TERM_MAP_MEMORY_ALLOC_FAIL)) { RAISE_ERROR(OUT_OF_MEMORY_ATOM); } else { @@ -5995,30 +5515,24 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } #endif } - if (!fail) { - NEXT_INSTRUCTION(next_off); - } break; } case OP_IS_TAGGED_TUPLE: { - int next_off = 1; uint32_t label; - DECODE_LABEL(label, code, i, next_off) + DECODE_LABEL(label, pc) term arg1; - DECODE_COMPACT_TERM(arg1, code, i, next_off) + DECODE_COMPACT_TERM(arg1, pc) uint32_t arity; - DECODE_LITERAL(arity, code, i, next_off) + DECODE_LITERAL(arity, pc) term tag_atom; - DECODE_ATOM(tag_atom, code, i, next_off) + DECODE_ATOM(tag_atom, pc) #ifdef IMPL_EXECUTE_LOOP TRACE("is_tagged_tuple/2, label=%u, arg1=%p, arity=%u, atom_id=%p\n", (unsigned) label, (void *) arg1, (unsigned) arity, (void *) tag_atom); - if (term_is_tuple(arg1) && ((uint32_t) term_get_tuple_arity(arg1) == arity) && (term_get_tuple_element(arg1, 0) == tag_atom)) { - NEXT_INSTRUCTION(next_off); - } else { - i = POINTER_TO_II(mod->labels[label]); + if (!(term_is_tuple(arg1) && ((uint32_t) term_get_tuple_arity(arg1) == arity) && (term_get_tuple_element(arg1, 0) == tag_atom))) { + pc = mod->labels[label]; } #endif @@ -6026,7 +5540,6 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("is_tagged_tuple/2\n"); UNUSED(label) UNUSED(arg1) - NEXT_INSTRUCTION(next_off); #endif break; @@ -6035,26 +5548,22 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) case OP_FCLEARERROR: { // This can be a noop as we raise from bifs TRACE("fclearerror/0\n"); - NEXT_INSTRUCTION(1); break; } case OP_FCHECKERROR: { - int next_off = 1; // This can be a noop as we raise from bifs int fail_label; - DECODE_LABEL(fail_label, code, i, next_off); - NEXT_INSTRUCTION(next_off); + DECODE_LABEL(fail_label, pc); break; } case OP_FMOVE: { - int next_off = 1; - if (IS_EXTENDED_FP_REGISTER(code, i, next_off)) { + if (IS_EXTENDED_FP_REGISTER(pc)) { int freg; - DECODE_FP_REGISTER(freg, code, i, next_off); + DECODE_FP_REGISTER(freg, pc); dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("fmove/2 fp%i, %c%i\n", freg, T_DEST_REG(dreg)); // Space should be available on heap as compiler added an allocate opcode @@ -6069,9 +5578,9 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif } else { term src_value; - DECODE_COMPACT_TERM(src_value, code, i, next_off); + DECODE_COMPACT_TERM(src_value, pc); int freg; - DECODE_FP_REGISTER(freg, code, i, next_off); + DECODE_FP_REGISTER(freg, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("fmove/2 %lx, fp%i\n", src_value, freg); context_ensure_fpregs(ctx); @@ -6083,17 +5592,14 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) UNUSED(freg) #endif } - - NEXT_INSTRUCTION(next_off); break; } case OP_FCONV: { - int next_off = 1; term src_value; - DECODE_COMPACT_TERM(src_value, code, i, next_off); + DECODE_COMPACT_TERM(src_value, pc); int freg; - DECODE_FP_REGISTER(freg, code, i, next_off); + DECODE_FP_REGISTER(freg, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("fconv/2 %lx, fp%i\n", src_value, freg); @@ -6106,8 +5612,6 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) UNUSED(freg) UNUSED(src_value) #endif - - NEXT_INSTRUCTION(next_off); break; } @@ -6115,13 +5619,12 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #ifdef HAVE_PRAGMA_STDC_FENV_ACCESS #pragma STDC FENV_ACCESS ON #endif - int next_off = 1; int fail_label; - DECODE_LABEL(fail_label, code, i, next_off); + DECODE_LABEL(fail_label, pc); int freg1, freg2, freg3; - DECODE_FP_REGISTER(freg1, code, i, next_off); - DECODE_FP_REGISTER(freg2, code, i, next_off); - DECODE_FP_REGISTER(freg3, code, i, next_off); + DECODE_FP_REGISTER(freg1, pc); + DECODE_FP_REGISTER(freg2, pc); + DECODE_FP_REGISTER(freg3, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("fadd/3 fp%i, fp%i, fp%i\n", freg1, freg2, freg3); #ifdef HAVE_PRAGMA_STDC_FENV_ACCESS @@ -6146,8 +5649,6 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) UNUSED(freg2) UNUSED(freg3) #endif - - NEXT_INSTRUCTION(next_off); break; } @@ -6155,13 +5656,12 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #ifdef HAVE_PRAGMA_STDC_FENV_ACCESS #pragma STDC FENV_ACCESS ON #endif - int next_off = 1; int fail_label; - DECODE_LABEL(fail_label, code, i, next_off); + DECODE_LABEL(fail_label, pc); int freg1, freg2, freg3; - DECODE_FP_REGISTER(freg1, code, i, next_off); - DECODE_FP_REGISTER(freg2, code, i, next_off); - DECODE_FP_REGISTER(freg3, code, i, next_off); + DECODE_FP_REGISTER(freg1, pc); + DECODE_FP_REGISTER(freg2, pc); + DECODE_FP_REGISTER(freg3, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("fsub/3 fp%i, fp%i, fp%i\n", freg1, freg2, freg3); #ifdef HAVE_PRAGMA_STDC_FENV_ACCESS @@ -6186,8 +5686,6 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) UNUSED(freg2) UNUSED(freg3) #endif - - NEXT_INSTRUCTION(next_off); break; } @@ -6195,13 +5693,12 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #ifdef HAVE_PRAGMA_STDC_FENV_ACCESS #pragma STDC FENV_ACCESS ON #endif - int next_off = 1; int fail_label; - DECODE_LABEL(fail_label, code, i, next_off); + DECODE_LABEL(fail_label, pc); int freg1, freg2, freg3; - DECODE_FP_REGISTER(freg1, code, i, next_off); - DECODE_FP_REGISTER(freg2, code, i, next_off); - DECODE_FP_REGISTER(freg3, code, i, next_off); + DECODE_FP_REGISTER(freg1, pc); + DECODE_FP_REGISTER(freg2, pc); + DECODE_FP_REGISTER(freg3, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("fmul/3 fp%i, fp%i, fp%i\n", freg1, freg2, freg3); #ifdef HAVE_PRAGMA_STDC_FENV_ACCESS @@ -6226,8 +5723,6 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) UNUSED(freg2) UNUSED(freg3) #endif - - NEXT_INSTRUCTION(next_off); break; } @@ -6235,13 +5730,12 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #ifdef HAVE_PRAGMA_STDC_FENV_ACCESS #pragma STDC FENV_ACCESS ON #endif - int next_off = 1; int fail_label; - DECODE_LABEL(fail_label, code, i, next_off); + DECODE_LABEL(fail_label, pc); int freg1, freg2, freg3; - DECODE_FP_REGISTER(freg1, code, i, next_off); - DECODE_FP_REGISTER(freg2, code, i, next_off); - DECODE_FP_REGISTER(freg3, code, i, next_off); + DECODE_FP_REGISTER(freg1, pc); + DECODE_FP_REGISTER(freg2, pc); + DECODE_FP_REGISTER(freg3, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("fdiv/3 fp%i, fp%i, fp%i\n", freg1, freg2, freg3); #ifdef HAVE_PRAGMA_STDC_FENV_ACCESS @@ -6266,18 +5760,15 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) UNUSED(freg2) UNUSED(freg3) #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_FNEGATE: { - int next_off = 1; int fail_label; - DECODE_LABEL(fail_label, code, i, next_off); + DECODE_LABEL(fail_label, pc); int freg1, freg2; - DECODE_FP_REGISTER(freg1, code, i, next_off); - DECODE_FP_REGISTER(freg2, code, i, next_off); + DECODE_FP_REGISTER(freg1, pc); + DECODE_FP_REGISTER(freg2, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("fnegate/2 fp%i, fp%i\n", freg1, freg2); context_ensure_fpregs(ctx); @@ -6289,13 +5780,10 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) UNUSED(freg1) UNUSED(freg2) #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_BUILD_STACKTRACE: { - int next_off = 1; TRACE("build_stacktrace/0\n"); @@ -6304,19 +5792,15 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) x_regs[0] = stacktrace_build(ctx, &x_regs[0]); #endif - - NEXT_INSTRUCTION(next_off); - break; } #ifdef ENABLE_OTP21 case OP_GET_HD: { - int next_off = 1; term src_value; - DECODE_COMPACT_TERM(src_value, code, i, next_off) + DECODE_COMPACT_TERM(src_value, pc) dreg_t head_dreg; - DECODE_DEST_REGISTER(head_dreg, code, i, next_off); + DECODE_DEST_REGISTER(head_dreg, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("get_hd/2 %lx, %c%i\n", src_value, T_DEST_REG(head_dreg)); @@ -6330,17 +5814,14 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("get_hd/2\n"); UNUSED(src_value) #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_GET_TL: { - int next_off = 1; term src_value; - DECODE_COMPACT_TERM(src_value, code, i, next_off) + DECODE_COMPACT_TERM(src_value, pc) dreg_t tail_dreg; - DECODE_DEST_REGISTER(tail_dreg, code, i, next_off); + DECODE_DEST_REGISTER(tail_dreg, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("get_tl/2 %lx, %c%i\n", src_value, T_DEST_REG(tail_dreg)); @@ -6354,20 +5835,17 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) TRACE("get_tl/2\n"); UNUSED(src_value) #endif - - NEXT_INSTRUCTION(next_off); break; } #endif #ifdef ENABLE_OTP22 case OP_PUT_TUPLE2: { - int next_off = 1; dreg_unsafe_t dreg; - DECODE_DEST_REGISTER_UNSAFE(dreg, code, i, next_off); - DECODE_EXTENDED_LIST_TAG(code, i, next_off); + DECODE_DEST_REGISTER_UNSAFE(dreg, pc); + DECODE_EXTENDED_LIST_TAG(pc); uint32_t size; - DECODE_LITERAL(size, code, i, next_off) + DECODE_LITERAL(size, pc) TRACE("put_tuple2/2, size=%i\n", size); USED_BY_TRACE(size); @@ -6386,7 +5864,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) for (uint32_t j = 0; j < size; j++) { term element; - DECODE_COMPACT_TERM(element, code, i, next_off) + DECODE_COMPACT_TERM(element, pc) #ifdef IMPL_CODE_LOADER UNUSED(element); @@ -6400,19 +5878,16 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #ifdef IMPL_EXECUTE_LOOP WRITE_REGISTER_UNSAFE(dreg, t); #endif - - NEXT_INSTRUCTION(next_off); break; } #endif #ifdef ENABLE_OTP23 case OP_SWAP: { - int next_off = 1; dreg_t reg_a; - DECODE_DEST_REGISTER(reg_a, code, i, next_off); + DECODE_DEST_REGISTER(reg_a, pc); dreg_t reg_b; - DECODE_DEST_REGISTER(reg_b, code, i, next_off); + DECODE_DEST_REGISTER(reg_b, pc); TRACE("swap/2 a=%c%i, b=%c%i\n", T_DEST_REG(reg_a), T_DEST_REG(reg_b)); @@ -6423,8 +5898,6 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) WRITE_REGISTER(reg_a, b); WRITE_REGISTER(reg_b, a); #endif - - NEXT_INSTRUCTION(next_off); break; } @@ -6436,17 +5909,16 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } #endif - int next_off = 1; // fail since OTP 23 might be either 'no_fail', 'resume' or a fail label // we are ignoring this right now, but we might use it for future optimizations. term fail; - DECODE_COMPACT_TERM(fail, code, i, next_off); + DECODE_COMPACT_TERM(fail, pc); uint32_t live; - DECODE_LITERAL(live, code, i, next_off); + DECODE_LITERAL(live, pc); term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_CODE_LOADER TRACE("bs_start_match4/4\n"); @@ -6457,32 +5929,26 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) if (!(term_is_binary(src) || term_is_match_state(src))) { WRITE_REGISTER(dreg, src); - i = POINTER_TO_II(mod->labels[fail]); + pc = mod->labels[fail]; } else { term match_state = term_alloc_bin_match_state(src, 0, &ctx->heap); WRITE_REGISTER(dreg, match_state); - NEXT_INSTRUCTION(next_off); } #endif - - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); - #endif break; } #endif #ifdef ENABLE_OTP24 case OP_MAKE_FUN3: { - int next_off = 1; uint32_t fun_index; - DECODE_LITERAL(fun_index, code, i, next_off); + DECODE_LITERAL(fun_index, pc); dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); - DECODE_EXTENDED_LIST_TAG(code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); + DECODE_EXTENDED_LIST_TAG(pc); uint32_t numfree; - DECODE_LITERAL(numfree, code, i, next_off) + DECODE_LITERAL(numfree, pc) TRACE("make_fun3/3, fun_index=%i dreg=%c%i numfree=%i\n", fun_index, T_DEST_REG(dreg), numfree); #ifdef IMPL_EXECUTE_LOOP @@ -6496,7 +5962,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) for (uint32_t j = 0; j < numfree; j++) { term arg; - DECODE_COMPACT_TERM(arg, code, i, next_off); + DECODE_COMPACT_TERM(arg, pc); #ifdef IMPL_EXECUTE_LOOP boxed_func[3 + j] = arg; #endif @@ -6506,88 +5972,76 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) term fun = ((term) boxed_func) | TERM_BOXED_VALUE_TAG; WRITE_REGISTER(dreg, fun); #endif - NEXT_INSTRUCTION(next_off); break; } case OP_INIT_YREGS: { - int next_off = 1; - DECODE_EXTENDED_LIST_TAG(code, i, next_off); + DECODE_EXTENDED_LIST_TAG(pc); uint32_t size; - DECODE_LITERAL(size, code, i, next_off); + DECODE_LITERAL(size, pc); for (uint32_t j = 0; j < size; j++) { uint32_t target; - DECODE_YREG(target, code, i, next_off); + DECODE_YREG(target, pc); #ifdef IMPL_EXECUTE_LOOP ctx->e[target] = term_nil(); #endif } - NEXT_INSTRUCTION(next_off); break; } case OP_RECV_MARKER_BIND: { - int next_off = 1; dreg_t reg_a; - DECODE_DEST_REGISTER(reg_a, code, i, next_off); + DECODE_DEST_REGISTER(reg_a, pc); dreg_t reg_b; - DECODE_DEST_REGISTER(reg_b, code, i, next_off); + DECODE_DEST_REGISTER(reg_b, pc); TRACE("recv_marker_bind/2: reg1=%c%i reg2=%c%i\n", T_DEST_REG(reg_a), T_DEST_REG(reg_b)); - NEXT_INSTRUCTION(next_off); break; } case OP_RECV_MARKER_CLEAR: { - int next_off = 1; dreg_t reg_a; - DECODE_DEST_REGISTER(reg_a, code, i, next_off); + DECODE_DEST_REGISTER(reg_a, pc); TRACE("recv_marker_clean/1: reg1=%c%i\n", T_DEST_REG(reg_a)); - NEXT_INSTRUCTION(next_off); break; } case OP_RECV_MARKER_RESERVE: { - int next_off = 1; dreg_t reg_a; - DECODE_DEST_REGISTER(reg_a, code, i, next_off); + DECODE_DEST_REGISTER(reg_a, pc); TRACE("recv_marker_reserve/1: reg1=%c%i\n", T_DEST_REG(reg_a)); #ifdef IMPL_EXECUTE_LOOP // Clear register to avoid any issue with GC WRITE_REGISTER(reg_a, term_nil()); #endif - NEXT_INSTRUCTION(next_off); break; } case OP_RECV_MARKER_USE: { - int next_off = 1; dreg_t reg_a; - DECODE_DEST_REGISTER(reg_a, code, i, next_off); + DECODE_DEST_REGISTER(reg_a, pc); TRACE("recv_marker_use/1: reg1=%c%i\n", T_DEST_REG(reg_a)); - NEXT_INSTRUCTION(next_off); break; } #endif #ifdef ENABLE_OTP25 case OP_BS_CREATE_BIN: { - int next_off = 1; uint32_t fail; - DECODE_LABEL(fail, code, i, next_off); + DECODE_LABEL(fail, pc); uint32_t alloc; - DECODE_LITERAL(alloc, code, i, next_off); + DECODE_LITERAL(alloc, pc); uint32_t live; - DECODE_LITERAL(live, code, i, next_off); + DECODE_LITERAL(live, pc); uint32_t unit; - DECODE_LITERAL(unit, code, i, next_off); + DECODE_LITERAL(unit, pc); dreg_unsafe_t dreg; - DECODE_DEST_REGISTER_UNSAFE(dreg, code, i, next_off); + DECODE_DEST_REGISTER_UNSAFE(dreg, pc); TRACE("bs_create_bin/6 fail=%i, alloc=%i live=%i unit=%i dreg=%c%i\n", fail, alloc, live, unit, T_DEST_REG_UNSAFE(dreg)); - DECODE_EXTENDED_LIST_TAG(code, i, next_off); + DECODE_EXTENDED_LIST_TAG(pc); uint32_t list_len; - DECODE_LITERAL(list_len, code, i, next_off); + DECODE_LITERAL(list_len, pc); #ifdef IMPL_EXECUTE_LOOP - int list_off = next_off; + const uint8_t* list_pc = pc; #endif size_t nb_segments = list_len / 6; #ifdef IMPL_CODE_LOADER @@ -6606,17 +6060,17 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif for (size_t j = 0; j < nb_segments; j++) { term atom_type; - DECODE_ATOM(atom_type, code, i, next_off); + DECODE_ATOM(atom_type, pc); int seg; - DECODE_LITERAL(seg, code, i, next_off); + DECODE_LITERAL(seg, pc); int segment_unit; - DECODE_LITERAL(segment_unit, code, i, next_off); + DECODE_LITERAL(segment_unit, pc); term flags; - DECODE_COMPACT_TERM(flags, code, i, next_off); + DECODE_COMPACT_TERM(flags, pc); term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); term size; - DECODE_COMPACT_TERM(size, code, i, next_off); + DECODE_COMPACT_TERM(size, pc); #ifdef IMPL_EXECUTE_LOOP avm_int_t src_value = 0; switch (atom_type) { @@ -6693,17 +6147,17 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) for (size_t j = 0; j < nb_segments; j++) { term atom_type; - DECODE_ATOM(atom_type, code, i, list_off); + DECODE_ATOM(atom_type, list_pc); int seg; - DECODE_LITERAL(seg, code, i, list_off); + DECODE_LITERAL(seg, list_pc); int segment_unit; - DECODE_LITERAL(segment_unit, code, i, list_off); + DECODE_LITERAL(segment_unit, list_pc); term flags; - DECODE_COMPACT_TERM(flags, code, i, list_off); + DECODE_COMPACT_TERM(flags, list_pc); term src; - DECODE_COMPACT_TERM(src, code, i, list_off); + DECODE_COMPACT_TERM(src, list_pc); term size; - DECODE_COMPACT_TERM(size, code, i, list_off); + DECODE_COMPACT_TERM(size, list_pc); size_t segment_size; avm_int_t flags_value = 0; avm_int_t src_value = 0; @@ -6832,22 +6286,19 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } WRITE_REGISTER_UNSAFE(dreg, t); #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_CALL_FUN2: { - int next_off = 1; term tag; - DECODE_COMPACT_TERM(tag, code, i, next_off) + DECODE_COMPACT_TERM(tag, pc) unsigned int args_count; - DECODE_LITERAL(args_count, code, i, next_off) + DECODE_LITERAL(args_count, pc) #ifdef IMPL_EXECUTE_LOOP - int fun_off = next_off; + const uint8_t *fun_pc = pc; #endif term fun; - DECODE_COMPACT_TERM(fun, code, i, next_off) + DECODE_COMPACT_TERM(fun, pc) TRACE("call_fun2/3, tag, args_count=%i, fun\n", args_count); USED_BY_TRACE(args_count); @@ -6859,24 +6310,18 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) RAISE_ERROR(OUT_OF_MEMORY_ATOM); } // Decode the function again after GC was possibly run - DECODE_COMPACT_TERM(fun, code, i, fun_off) + DECODE_COMPACT_TERM(fun, fun_pc) term new_error_tuple = term_alloc_tuple(2, &ctx->heap); term_put_tuple_element(new_error_tuple, 0, BADFUN_ATOM); term_put_tuple_element(new_error_tuple, 1, fun); RAISE_ERROR(new_error_tuple); } - CALL_FUN(fun, args_count, next_off) - #endif - - #ifdef IMPL_CODE_LOADER - NEXT_INSTRUCTION(next_off); + CALL_FUN(fun, args_count) #endif - break; } case OP_BADRECORD: { - int next_off = 1; TRACE("badrecord/1\n"); #ifdef IMPL_EXECUTE_LOOP @@ -6885,7 +6330,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) RAISE_ERROR(OUT_OF_MEMORY_ATOM); } term value; - DECODE_COMPACT_TERM(value, code, i, next_off) + DECODE_COMPACT_TERM(value, pc) term new_error_tuple = term_alloc_tuple(2, &ctx->heap); term_put_tuple_element(new_error_tuple, 0, BADRECORD_ATOM); term_put_tuple_element(new_error_tuple, 1, value); @@ -6894,8 +6339,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #ifdef IMPL_CODE_LOADER term value; - DECODE_COMPACT_TERM(value, code, i, next_off) - NEXT_INSTRUCTION(next_off); + DECODE_COMPACT_TERM(value, pc) #endif break; @@ -6904,15 +6348,14 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #ifdef ENABLE_OTP26 case OP_UPDATE_RECORD: { - int next_off = 1; #ifdef IMPL_CODE_LOADER TRACE("update_record/5\n"); #endif term hint; - DECODE_ATOM(hint, code, i, next_off); + DECODE_ATOM(hint, pc); int size; - DECODE_LITERAL(size, code, i, next_off); + DECODE_LITERAL(size, pc); #ifdef IMPL_EXECUTE_LOOP // MEMORY_NO_SHRINK because update_record is classified as put in beam_ssa_codegen.erl if (UNLIKELY(memory_ensure_free_opt(ctx, TUPLE_SIZE(size), MEMORY_NO_SHRINK) != MEMORY_GC_OK)) { @@ -6922,18 +6365,18 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) dst = term_alloc_tuple(size, &ctx->heap); #endif term src; - DECODE_COMPACT_TERM(src, code, i, next_off); + DECODE_COMPACT_TERM(src, pc); dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); #ifdef IMPL_EXECUTE_LOOP if (UNLIKELY(!term_is_tuple(src) || (size != term_get_tuple_arity(src)))) { fprintf(stderr, "update_record/5 !term_is_tuple(src) or size doesn't match\n"); AVM_ABORT(); } #endif - DECODE_EXTENDED_LIST_TAG(code, i, next_off); + DECODE_EXTENDED_LIST_TAG(pc); int list_len; - DECODE_LITERAL(list_len, code, i, next_off); + DECODE_LITERAL(list_len, pc); #ifdef IMPL_EXECUTE_LOOP TRACE("update_record/5 hint=%lu, size=%i, src=%p, dst=%p, updates_len=%d\n", hint, size, (void *)src, (void *)dst, list_len); bool reuse = hint == REUSE_ATOM; @@ -6943,9 +6386,9 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif for (int j = 0; j < list_len; j+=2) { int update_ix; - DECODE_LITERAL(update_ix, code, i, next_off); + DECODE_LITERAL(update_ix, pc); term update_value; - DECODE_COMPACT_TERM(update_value, code, i, next_off); + DECODE_COMPACT_TERM(update_value, pc); #ifdef IMPL_EXECUTE_LOOP if (reuse) { term old_value = term_get_tuple_element(dst, update_ix - 1); @@ -6967,42 +6410,39 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) WRITE_REGISTER(dreg, dst); } #endif - - NEXT_INSTRUCTION(next_off); break; } case OP_BS_MATCH: { - int next_off = 1; TRACE("bs_match/3\n"); int fail; - DECODE_LABEL(fail, code, i, next_off); + DECODE_LABEL(fail, pc); #ifdef IMPL_EXECUTE_LOOP - int match_off = next_off; + const uint8_t *match_pc = pc; #endif term match_state; - DECODE_COMPACT_TERM(match_state, code, i, next_off); + DECODE_COMPACT_TERM(match_state, pc); #ifdef IMPL_EXECUTE_LOOP VERIFY_IS_MATCH_STATE(match_state, "bs_match/3") term bs_bin = term_get_match_state_binary(match_state); size_t bs_offset = term_get_match_state_offset(match_state); #endif - DECODE_EXTENDED_LIST_TAG(code, i, next_off); + DECODE_EXTENDED_LIST_TAG(pc); int list_len; - DECODE_LITERAL(list_len, code, i, next_off); + DECODE_LITERAL(list_len, pc); int j = 0; while (j < list_len) { term command; - DECODE_ATOM(command, code, i, next_off); + DECODE_ATOM(command, pc); j++; switch (command) { case ENSURE_AT_LEAST_ATOM: { int stride; - DECODE_LITERAL(stride, code, i, next_off); + DECODE_LITERAL(stride, pc); j++; int unit; // TODO: check use of unit here - DECODE_LITERAL(unit, code, i, next_off); + DECODE_LITERAL(unit, pc); j++; #ifdef IMPL_EXECUTE_LOOP size_t bs_bin_size = term_binary_size(bs_bin); @@ -7020,7 +6460,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) case ENSURE_EXACTLY_ATOM: { int stride; - DECODE_LITERAL(stride, code, i, next_off); + DECODE_LITERAL(stride, pc); j++; #ifdef IMPL_EXECUTE_LOOP if (UNLIKELY(stride < 0)) { @@ -7038,20 +6478,20 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) case INTEGER_ATOM: { int live; - DECODE_LITERAL(live, code, i, next_off); + DECODE_LITERAL(live, pc); j++; term flags; - DECODE_COMPACT_TERM(flags, code, i, next_off); + DECODE_COMPACT_TERM(flags, pc); j++; #ifdef IMPL_EXECUTE_LOOP avm_int_t flags_value; DECODE_FLAGS_LIST(flags_value, flags, opcode) #endif term size; - DECODE_COMPACT_TERM(size, code, i, next_off); + DECODE_COMPACT_TERM(size, pc); j++; int unit; - DECODE_LITERAL(unit, code, i, next_off); + DECODE_LITERAL(unit, pc); j++; #ifdef IMPL_EXECUTE_LOOP // context_clean_registers(ctx, live); // TODO: check if needed @@ -7070,7 +6510,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) } #endif dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); j++; #ifdef IMPL_EXECUTE_LOOP WRITE_REGISTER(dreg, t); @@ -7081,20 +6521,20 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) case BINARY_ATOM: { int live; - DECODE_LITERAL(live, code, i, next_off); + DECODE_LITERAL(live, pc); j++; term flags; - DECODE_COMPACT_TERM(flags, code, i, next_off); + DECODE_COMPACT_TERM(flags, pc); j++; #ifdef IMPL_EXECUTE_LOOP avm_int_t flags_value; DECODE_FLAGS_LIST(flags_value, flags, opcode) #endif int size; - DECODE_LITERAL(size, code, i, next_off); + DECODE_LITERAL(size, pc); j++; int unit; - DECODE_LITERAL(unit, code, i, next_off); + DECODE_LITERAL(unit, pc); j++; #ifdef IMPL_EXECUTE_LOOP // context_clean_registers(ctx, live); // TODO: check if needed @@ -7112,13 +6552,13 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) RAISE_ERROR(OUT_OF_MEMORY_ATOM); } // re-compute match_state as GC could have moved it - int temp = match_off; - DECODE_COMPACT_TERM(match_state, code, i, temp); + const uint8_t *temp = match_pc; + DECODE_COMPACT_TERM(match_state, temp); bs_bin = term_get_match_state_binary(match_state); term t = term_maybe_create_sub_binary(bs_bin, bs_offset / 8, matched_bits / 8, &ctx->heap, ctx->global); #endif dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); j++; #ifdef IMPL_EXECUTE_LOOP WRITE_REGISTER(dreg, t); @@ -7129,10 +6569,10 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) case GET_TAIL_ATOM: { int live; - DECODE_LITERAL(live, code, i, next_off); + DECODE_LITERAL(live, pc); j++; int unit; - DECODE_LITERAL(unit, code, i, next_off); + DECODE_LITERAL(unit, pc); j++; #ifdef IMPL_EXECUTE_LOOP // context_clean_registers(ctx, live); // TODO: check if needed @@ -7147,13 +6587,13 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) RAISE_ERROR(OUT_OF_MEMORY_ATOM); } // re-compute match_state as GC could have moved it - int temp = match_off; - DECODE_COMPACT_TERM(match_state, code, i, temp); + const uint8_t *temp = match_pc; + DECODE_COMPACT_TERM(match_state, temp); bs_bin = term_get_match_state_binary(match_state); term t = term_maybe_create_sub_binary(bs_bin, bs_offset / 8, tail_bits / 8, &ctx->heap, ctx->global); #endif dreg_t dreg; - DECODE_DEST_REGISTER(dreg, code, i, next_off); + DECODE_DEST_REGISTER(dreg, pc); j++; #ifdef IMPL_EXECUTE_LOOP WRITE_REGISTER(dreg, t); @@ -7164,13 +6604,13 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) case EQUAL_COLON_EQUAL_ATOM: { term live; - DECODE_COMPACT_TERM(live, code, i, next_off); + DECODE_COMPACT_TERM(live, pc); j++; int size; - DECODE_LITERAL(size, code, i, next_off); + DECODE_LITERAL(size, pc); j++; avm_int_t pattern_value; - DECODE_LITERAL(pattern_value, code, i, next_off); + DECODE_LITERAL(pattern_value, pc); j++; #ifdef IMPL_EXECUTE_LOOP union maybe_unsigned_int64 matched_value; @@ -7190,7 +6630,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) case SKIP_ATOM: { int stride; - DECODE_LITERAL(stride, code, i, next_off); + DECODE_LITERAL(stride, pc); j++; #ifdef IMPL_EXECUTE_LOOP bs_offset += stride; @@ -7201,7 +6641,7 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) default: fprintf(stderr, "bs_match/3: undecoded command: %i, j = %d, list_len = %d\n\n", (int) term_to_atom_index(command), j, list_len); #ifdef IMPL_EXECUTE_LOOP - fprintf(stderr, "failed at %" PRIuPTR "\n", i); + fprintf(stderr, "failed at %" PRIuPTR "\n", pc - code); #endif AVM_ABORT(); @@ -7210,7 +6650,6 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) term_set_match_state_offset(match_state, bs_offset); #endif } - NEXT_INSTRUCTION(next_off); break; #ifdef IMPL_EXECUTE_LOOP @@ -7222,11 +6661,10 @@ HOT_FUNC int scheduler_entry_point(GlobalContext *glb) #endif default: - printf("Undecoded opcode: %i\n", code[i]); + printf("Undecoded opcode: %i\n", *pc); #ifdef IMPL_EXECUTE_LOOP - fprintf(stderr, "failed at %u\n", (unsigned int) i); + fprintf(stderr, "failed at %" PRIuPTR "\n", pc - code); #endif - AVM_ABORT(); return 1; } diff --git a/src/libAtomVM/stacktrace.c b/src/libAtomVM/stacktrace.c index 228362a25..41ca45f22 100644 --- a/src/libAtomVM/stacktrace.c +++ b/src/libAtomVM/stacktrace.c @@ -58,7 +58,7 @@ static void cp_to_mod_lbl_off(term cp, Context *ctx, Module **cp_mod, int *label int labels_count = ENDIAN_SWAP_32(mod->code->labels); int i = 1; - uint8_t *l = mod->labels[1]; + const uint8_t *l = mod->labels[1]; while (*mod_offset > l - code) { i++; if (i >= labels_count) { @@ -71,7 +71,7 @@ static void cp_to_mod_lbl_off(term cp, Context *ctx, Module **cp_mod, int *label } *label = i - 1; - *l_off = *mod_offset - ((uint8_t *) mod->labels[*label] - code); + *l_off = *mod_offset - (mod->labels[*label] - code); } static bool is_module_member(Module *mod, Module **mods, unsigned long len) @@ -131,7 +131,7 @@ term stacktrace_create_raw(Context *ctx, Module *mod, int current_offset, term e Module *cl_mod = globalcontext_get_module_by_index(ctx->global, module_index); uint8_t *code = &cl_mod->code->code[0]; - int mod_offset = ((uint8_t *) cl_mod->labels[label] - code); + int mod_offset = cl_mod->labels[label] - code; if (!(prev_mod == cl_mod && mod_offset == prev_mod_offset)) { ++num_frames; @@ -205,7 +205,7 @@ term stacktrace_create_raw(Context *ctx, Module *mod, int current_offset, term e int label = term_to_catch_label_and_module(*ct, &module_index); Module *cl_mod = globalcontext_get_module_by_index(ctx->global, module_index); uint8_t *code = &cl_mod->code->code[0]; - int mod_offset = ((uint8_t *) cl_mod->labels[label] - code); + int mod_offset = cl_mod->labels[label] - code; if (!(prev_mod == cl_mod && mod_offset == prev_mod_offset)) {