Add support for static syscalls (#296)

This adds support for static, relocation-less syscalls using the same mechanism
as upstream BPF:

- CALL_IMMs with src=0 are considered syscalls, where the .imm field is the
  murmur32 hash of the syscall name
- CALL_IMMs with src=1 are considered bpf-to-bpf calls

src/assembler.rs

@@ -326,7 +326,7 @@ pub fn assemble<E: UserDefinedError, I: 'static + InstructionMeter>(
                                 &label,
                                 Some(target_pc),
                             )?;
-                            insn(opc, 0, 0, 0, hash as i32 as i64)
+                            insn(opc, 0, 1, 0, hash as i32 as i64)
                         }
                         (CallReg, [Register(dst)]) => insn(opc, 0, 0, 0, *dst),
                         (JumpConditional, [Register(dst), Register(src), Label(label)]) => insn(
@@ -359,7 +359,7 @@ pub fn assemble<E: UserDefinedError, I: 'static + InstructionMeter>(
                                 label,
                                 None,
                             )?;
-                            insn(opc, 0, 0, 0, hash as i32 as i64)
+                            insn(opc, 0, 1, 0, hash as i32 as i64)
                         }
                         (Endian(size), [Register(dst)]) => insn(opc, *dst, 0, 0, size),
                         (LoadImm, [Register(dst), Integer(imm)]) => {

src/elf.rs

@@ -561,7 +561,10 @@ impl<E: UserDefinedError, I: InstructionMeter> Executable<E, I> {
             .ok_or(ElfError::ValueOutOfBounds)?;
         for i in 0..instruction_count {
             let mut insn = ebpf::get_insn(elf_bytes, i);
-            if insn.opc == ebpf::CALL_IMM && insn.imm != -1 {
+            if insn.opc == ebpf::CALL_IMM
+                && insn.imm != -1
+                && !(config.static_syscalls && insn.src == 0)
+            {
                 let target_pc = (i as isize)
                     .saturating_add(1)
                     .saturating_add(insn.imm as isize);
@@ -1794,6 +1797,25 @@ mod test {
             .expect("validation failed");
     }
 
+    #[test]
+    #[should_panic(expected = r#"validation failed: RelativeJumpOutOfBounds(29)"#)]
+    fn test_static_syscall_disabled() {
+        let elf_bytes =
+            std::fs::read("tests/elfs/syscall_static_unknown.so").expect("failed to read elf file");
+
+        // when config.static_syscalls=false, all CALL_IMMs are treated as relative
+        // calls for backwards compatibility
+        ElfExecutable::load(
+            Config {
+                static_syscalls: false,
+                ..Config::default()
+            },
+            &elf_bytes,
+            syscall_registry(),
+        )
+        .expect("validation failed");
+    }
+
     #[cfg(all(not(windows), target_arch = "x86_64"))]
     #[test]
     fn test_size() {

src/jit.rs

@@ -1329,36 +1329,57 @@ impl JitCompiler {
                     // For JIT, syscalls MUST be registered at compile time. They can be
                     // updated later, but not created after compiling (we need the address of the
                     // syscall function in the JIT-compiled program).
-                    if let Some(syscall) = executable.get_syscall_registry().lookup_syscall(insn.imm as u32) {
-                        if self.config.enable_instruction_meter {
-                            emit_validate_and_profile_instruction_count(self, true, Some(0))?;
+
+                    let mut resolved = false;
+                    let (syscalls, calls) = if self.config.static_syscalls {
+                        (insn.src == 0, insn.src != 0)
+                    } else {
+                        (true, true)
+                    };
+
+                    if syscalls {
+                        if let Some(syscall) = executable.get_syscall_registry().lookup_syscall(insn.imm as u32) {
+                            if self.config.enable_instruction_meter {
+                                emit_validate_and_profile_instruction_count(self, true, Some(0))?;
+                            }
+                            X86Instruction::load_immediate(OperandSize::S64, R11, syscall.function as *const u8 as i64).emit(self)?;
+                            X86Instruction::load(OperandSize::S64, R10, RAX, X86IndirectAccess::Offset((SYSCALL_CONTEXT_OBJECTS_OFFSET + syscall.context_object_slot) as i32 * 8 + self.program_argument_key)).emit(self)?;
+                            emit_call(self, TARGET_PC_SYSCALL)?;
+                            if self.config.enable_instruction_meter {
+                                emit_undo_profile_instruction_count(self, 0)?;
+                            }
+                            // Throw error if the result indicates one
+                            X86Instruction::cmp_immediate(OperandSize::S64, R11, 0, Some(X86IndirectAccess::Offset(0))).emit(self)?;
+                            X86Instruction::load_immediate(OperandSize::S64, R11, self.pc as i64).emit(self)?;
+                            emit_jcc(self, 0x85, TARGET_PC_RUST_EXCEPTION)?;
+
+                            resolved = true;
                         }
-                        X86Instruction::load_immediate(OperandSize::S64, R11, syscall.function as *const u8 as i64).emit(self)?;
-                        X86Instruction::load(OperandSize::S64, R10, RAX, X86IndirectAccess::Offset((SYSCALL_CONTEXT_OBJECTS_OFFSET + syscall.context_object_slot) as i32 * 8 + self.program_argument_key)).emit(self)?;
-                        emit_call(self, TARGET_PC_SYSCALL)?;
-                        if self.config.enable_instruction_meter {
-                            emit_undo_profile_instruction_count(self, 0)?;
+                    }
+
+                    if calls {
+                        if let Some(target_pc) = executable.lookup_bpf_function(insn.imm as u32) {
+                            emit_bpf_call(self, Value::Constant64(target_pc as i64, false))?;
+                            resolved = true;
+                        }
+                    }
+
+                    if !resolved {
+                        if self.config.disable_unresolved_symbols_at_runtime {
+                            X86Instruction::load_immediate(OperandSize::S64, R11, self.pc as i64).emit(self)?;
+                            emit_jmp(self, TARGET_PC_CALL_UNSUPPORTED_INSTRUCTION)?;
+                        } else {
+                            emit_validate_instruction_count(self, true, Some(self.pc))?;
+                            // executable.report_unresolved_symbol(self.pc)?;
+                            // Workaround for unresolved symbols in ELF: Report error at runtime instead of compiletime
+                            emit_rust_call(self, Value::Constant64(Executable::<E, I>::report_unresolved_symbol as *const u8 as i64, false), &[
+                                Argument { index: 2, value: Value::Constant64(self.pc as i64, false) },
+                                Argument { index: 1, value: Value::Constant64(&*executable.as_ref() as *const _ as i64, false) },
+                                Argument { index: 0, value: Value::RegisterIndirect(RBP, slot_on_environment_stack(self, EnvironmentStackSlot::OptRetValPtr), false) },
+                            ], None, true)?;
+                            X86Instruction::load_immediate(OperandSize::S64, R11, self.pc as i64).emit(self)?;
+                            emit_jmp(self, TARGET_PC_RUST_EXCEPTION)?;
                         }
-                        // Throw error if the result indicates one
-                        X86Instruction::cmp_immediate(OperandSize::S64, R11, 0, Some(X86IndirectAccess::Offset(0))).emit(self)?;
-                        X86Instruction::load_immediate(OperandSize::S64, R11, self.pc as i64).emit(self)?;
-                        emit_jcc(self, 0x85, TARGET_PC_RUST_EXCEPTION)?;
-                    } else if let Some(target_pc) = executable.lookup_bpf_function(insn.imm as u32) {
-                        emit_bpf_call(self, Value::Constant64(target_pc as i64, false))?;
-                    } else if self.config.disable_unresolved_symbols_at_runtime {
-                        X86Instruction::load_immediate(OperandSize::S64, R11, self.pc as i64).emit(self)?;
-                        emit_jmp(self, TARGET_PC_CALL_UNSUPPORTED_INSTRUCTION)?;
-                    } else {
-                        emit_validate_instruction_count(self, true, Some(self.pc))?;
-                        // executable.report_unresolved_symbol(self.pc)?;
-                        // Workaround for unresolved symbols in ELF: Report error at runtime instead of compiletime
-                        emit_rust_call(self, Value::Constant64(Executable::<E, I>::report_unresolved_symbol as *const u8 as i64, false), &[
-                            Argument { index: 2, value: Value::Constant64(self.pc as i64, false) },
-                            Argument { index: 1, value: Value::Constant64(&*executable.as_ref() as *const _ as i64, false) },
-                            Argument { index: 0, value: Value::RegisterIndirect(RBP, slot_on_environment_stack(self, EnvironmentStackSlot::OptRetValPtr), false) },
-                        ], None, true)?;
-                        X86Instruction::load_immediate(OperandSize::S64, R11, self.pc as i64).emit(self)?;
-                        emit_jmp(self, TARGET_PC_RUST_EXCEPTION)?;
                     }
                 },
                 ebpf::CALL_REG  => {

src/vm.rs

@@ -231,6 +231,8 @@ pub struct Config {
     pub enable_sdiv: bool,
     /// Avoid copying read only sections when possible
     pub optimize_rodata: bool,
+    /// Support syscalls via pseudo calls (insn.src = 0)
+    pub static_syscalls: bool,
 }
 
 impl Config {
@@ -261,6 +263,7 @@ impl Default for Config {
             dynamic_stack_frames: true,
             enable_sdiv: true,
             optimize_rodata: true,
+            static_syscalls: true,
         }
     }
 }
@@ -711,9 +714,11 @@ impl<'a, E: UserDefinedError, I: InstructionMeter> EbpfVm<'a, E, I> {
         }
 
         // Check config outside of the instruction loop
-        let instruction_meter_enabled = self.executable.get_config().enable_instruction_meter;
-        let instruction_tracing_enabled = self.executable.get_config().enable_instruction_tracing;
-        let dynamic_stack_frames = self.executable.get_config().dynamic_stack_frames;
+        let config = self.executable.get_config();
+        let instruction_meter_enabled = config.enable_instruction_meter;
+        let instruction_tracing_enabled = config.enable_instruction_tracing;
+        let dynamic_stack_frames = config.dynamic_stack_frames;
+        let static_syscalls = config.static_syscalls;
 
         // Loop on instructions
         let entry = self.executable.get_entrypoint_instruction_offset()?;
@@ -1031,36 +1036,57 @@ impl<'a, E: UserDefinedError, I: InstructionMeter> EbpfVm<'a, E, I> {
                 // Do not delegate the check to the verifier, since registered functions can be
                 // changed after the program has been verified.
                 ebpf::CALL_IMM => {
-                    if let Some(syscall) = self.executable.get_syscall_registry().lookup_syscall(insn.imm as u32) {
-                        if instruction_meter_enabled {
-                            let _ = instruction_meter.consume(self.last_insn_count);
+                    let mut resolved = false;
+                    let (syscalls, calls) = if static_syscalls {
+                        (insn.src == 0, insn.src != 0)
+                    } else {
+                        (true, true)
+                    };
+
+                    if syscalls {
+                        if let Some(syscall) = self.executable.get_syscall_registry().lookup_syscall(insn.imm as u32) {
+                            resolved = true;
+
+                            if instruction_meter_enabled {
+                                let _ = instruction_meter.consume(self.last_insn_count);
+                            }
+                            total_insn_count += self.last_insn_count;
+                            self.last_insn_count = 0;
+                            let mut result: ProgramResult<E> = Ok(0);
+                            (unsafe { std::mem::transmute::<u64, SyscallFunction::<E, *mut u8>>(syscall.function) })(
+                                self.syscall_context_objects[SYSCALL_CONTEXT_OBJECTS_OFFSET + syscall.context_object_slot],
+                                reg[1],
+                                reg[2],
+                                reg[3],
+                                reg[4],
+                                reg[5],
+                                &self.memory_mapping,
+                                &mut result,
+                            );
+                            reg[0] = result?;
+                            if instruction_meter_enabled {
+                                remaining_insn_count = instruction_meter.get_remaining();
+                            }
                         }
-                        total_insn_count += self.last_insn_count;
-                        self.last_insn_count = 0;
-                        let mut result: ProgramResult<E> = Ok(0);
-                        (unsafe { std::mem::transmute::<u64, SyscallFunction::<E, *mut u8>>(syscall.function) })(
-                            self.syscall_context_objects[SYSCALL_CONTEXT_OBJECTS_OFFSET + syscall.context_object_slot],
-                            reg[1],
-                            reg[2],
-                            reg[3],
-                            reg[4],
-                            reg[5],
-                            &self.memory_mapping,
-                            &mut result,
-                        );
-                        reg[0] = result?;
-                        if instruction_meter_enabled {
-                            remaining_insn_count = instruction_meter.get_remaining();
+                    }
+
+                    if calls {
+                        if let Some(target_pc) = self.executable.lookup_bpf_function(insn.imm as u32) {
+                            resolved = true;
+
+                            // make BPF to BPF call
+                            reg[ebpf::FRAME_PTR_REG] =
+                                self.stack.push(&reg[ebpf::FIRST_SCRATCH_REG..ebpf::FIRST_SCRATCH_REG + ebpf::SCRATCH_REGS], next_pc)?;
+                            next_pc = self.check_pc(pc, target_pc)?;
+                        }
+                    }
+
+                    if !resolved {
+                        if config.disable_unresolved_symbols_at_runtime {
+                            return Err(EbpfError::UnsupportedInstruction(pc + ebpf::ELF_INSN_DUMP_OFFSET));
+                        } else {
+                            self.executable.report_unresolved_symbol(pc)?;
                         }
-                    } else if let Some(target_pc) = self.executable.lookup_bpf_function(insn.imm as u32) {
-                        // make BPF to BPF call
-                        reg[ebpf::FRAME_PTR_REG] =
-                            self.stack.push(&reg[ebpf::FIRST_SCRATCH_REG..ebpf::FIRST_SCRATCH_REG + ebpf::SCRATCH_REGS], next_pc)?;
-                        next_pc = self.check_pc(pc, target_pc)?;
-                    } else if self.executable.get_config().disable_unresolved_symbols_at_runtime {
-                        return Err(EbpfError::UnsupportedInstruction(pc + ebpf::ELF_INSN_DUMP_OFFSET));
-                    } else {
-                        self.executable.report_unresolved_symbol(pc)?;
                     }
                 }
 

tests/assembler.rs

@@ -130,7 +130,7 @@ fn test_call_reg() {
 fn test_call_imm() {
     assert_eq!(
         asm("call 300"),
-        Ok(vec![insn(0, ebpf::CALL_IMM, 0, 0, 0, -1090069135)])
+        Ok(vec![insn(0, ebpf::CALL_IMM, 0, 1, 0, -1090069135)])
     );
 }
 

tests/elfs/elf.ld

@@ -9,11 +9,11 @@ SECTIONS
 {
   . = SIZEOF_HEADERS;
   .text : { *(.text) } :text
-  .rodata : { *(.rodata) } :rodata
+  .rodata : { *(.rodata*) } :rodata
   .dynamic : { *(.dynamic) } :dynamic
   .dynsym : { *(.dynsym) } :dynamic
   .dynstr : { *(.dynstr) } :dynamic
   .gnu.hash : { *(.gnu.hash) } :dynamic
   .rel.dyn : { *(.rel.dyn) } :dynamic
   .hash : { *(.hash) } :dynamic
-}
+}

tests/elfs/elfs.sh

@@ -54,3 +54,11 @@ rm bss_section.o
 "$LLVM_DIR"clang -Werror -target bpf -O2 -fno-builtin -fPIC -o rodata.o -c rodata.c
 "$LLVM_DIR"ld.lld -z notext -shared --Bdynamic -entry entrypoint --script elf.ld -o rodata.so rodata.o
 rm rodata.o
+
+"$LLVM_DIR"clang -Werror -target bpf -O2 -fno-builtin -fPIC -o syscall_static.o -c syscall_static.c
+"$LLVM_DIR"ld.lld -z notext -shared --Bdynamic -entry entrypoint --script elf.ld -o syscall_static.so syscall_static.o
+rm syscall_static.o
+
+"$LLVM_DIR"clang -Werror -target bpf -O2 -fno-builtin -fPIC -o syscall_static_unknown.o -c syscall_static_unknown.c
+"$LLVM_DIR"ld.lld -z notext -shared --Bdynamic -entry entrypoint --script elf.ld -o syscall_static_unknown.so syscall_static_unknown.o
+rm syscall_static_unknown.o

tests/elfs/syscall_static.c

@@ -0,0 +1,11 @@
+typedef unsigned long int uint64_t;
+typedef unsigned char uint8_t;
+
+// 1811268606 = murmur32("log");
+static void (*log)(const char*, uint64_t) = (void *) 1811268606;
+
+extern uint64_t entrypoint(const uint8_t *input) {
+  log("foo\n", 4);
+
+  return 0;
+}

tests/elfs/syscall_static_unknown.c

@@ -0,0 +1,10 @@
+typedef unsigned long int uint64_t;
+typedef unsigned char uint8_t;
+
+static void (*i_dont_exist)() = (void *) 42;
+
+extern uint64_t entrypoint(const uint8_t *input) {
+  i_dont_exist();
+
+  return 0;
+}