experiment: value quota based idl decoder limiting (#4657)

Simplifies #4624 to a simple linear limit on the number of decoded values as a function of decoded payload size, instead of using two linear functions on perfcounter (simulated or real) and allocation counter. The function is: value_quota(blob) : Nat64 = blob.size() * (numerator/denominator) + bias where blob is the candid blob to be decoded, and `numerator` (default 1), `denominator` (default 1) and `bias` (default 1024) are `Nat32s`. Much simpler than #4624 and doesn't depend on vagaries of instruction metering and byte allocation which varies with gc and compiler options, but is it good enough? The constants can be (globally) modified/inspected using prims (Prim.getCandidLimits/Prim.setCandidLimits) which will need to get exposed in base eventually. The quota is decremented on every call to deserialise or skip a value in vanilla candid mode (destabilization is not metered). The quota is eagerly checked before deserializing or skipping arrays. One possible refinement would be to combine the value quota with a memory quota (though the latter would still vary with gc flavour and perhaps word-size unless we count logical words) - [x] Disable for destabilization (iff Registers.get_rel_buf_opt is zero) - [x] Port new candid spacebomb test suite to drun-tests, to test against real perf counter provided by drun. - [x] Bump candid dependency to most recent - [x] Pass new spacebomb tests, both in candid test suite on wasmtime using value counter.
dfinity · Aug 16, 2024 · b03a4d6 · b03a4d6
1 parent 367145d
commit b03a4d6
Show file tree

Hide file tree

Showing 26 changed files with 581 additions and 51 deletions.
diff --git a/Changelog.md b/Changelog.md
@@ -1,5 +1,21 @@
 # Motoko compiler changelog
 
+* motoko (`moc`)
+
+  * Candid decoding: impose an upper limit on the number of values decoded or skipped in a single candid payload,
+    as a linear function, `max_values`, of binary payload size.
+
+    ```
+      max_values(blob) = (blob.size() * numerator)/denominator + bias
+    ```
+
+    The current default settings are `{numerator = 1; denominator = 1; bias = 1024 }`, allowing a maximum
+    of 1024 values plus one additional value per byte in the payload.
+
+    While hopefully not required, the constant factors can be read/modified using system functions:
+    *  Prim.setCandidLimits: `<system>{numerator : Nat32;  denominator : Nat32; bias : Nat32 } -> ()`
+    *  Prim.getCandidLimits: `<system>() -> {numerator : Nat32; denominator : Nat32; bias : Nat32 }`
+
 ## 0.12.1 (2024-08-08)
 
 * motoko (`moc`)

diff --git a/nix/sources.json b/nix/sources.json
@@ -6,10 +6,10 @@
         "homepage": "",
         "owner": "dfinity",
         "repo": "candid",
-        "rev": "331217bae379cbebfa531a140f2186c99fae1425",
-        "sha256": "095w2a4lxy2gd7vfjxn7jszm4x3srw8xlxb1zzd096y6h047rxlj",
+        "rev": "34b4eb0b581bbf04902e20bf1370e3a293d1956f",
+        "sha256": "1gr49p938hzm8fq4r3n7j2lzfj0hmah5sb411sma24plnmwy7ljx",
         "type": "tarball",
-        "url": "https://github.com/dfinity/candid/archive/331217bae379cbebfa531a140f2186c99fae1425.tar.gz",
+        "url": "https://github.com/dfinity/candid/archive/34b4eb0b581bbf04902e20bf1370e3a293d1956f.tar.gz",
         "url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
     },
     "esm": {

diff --git a/rts/motoko-rts/src/idl.rs b/rts/motoko-rts/src/idl.rs
@@ -11,6 +11,11 @@ use core::cmp::min;
 
 use motoko_rts_macros::ic_mem_fn;
 
+extern "C" {
+    // check instruction decoding limit, exported by moc
+    pub fn idl_limit_check(decrement: bool, value_count: u64);
+}
+
 //
 // IDL constants
 //
@@ -306,6 +311,7 @@ unsafe fn skip_any_vec(buf: *mut Buf, typtbl: *mut *mut u8, t: i32, count: u32)
     if count == 0 {
         return;
     }
+    idl_limit_check(false, count as u64);
     let ptr_before = (*buf).ptr;
     skip_any(buf, typtbl, t, 0);
     let ptr_after = (*buf).ptr;
@@ -314,6 +320,7 @@ unsafe fn skip_any_vec(buf: *mut Buf, typtbl: *mut *mut u8, t: i32, count: u32)
         // makes no progress. No point in calling it over and over again.
         // (This is easier to detect this way than by analyzing the type table,
         // where we’d have to chase single-field-records.)
+        idl_limit_check(true, (count - 1) as u64);
         return;
     }
     for _ in 1..count {
@@ -332,6 +339,8 @@ unsafe extern "C" fn skip_any(buf: *mut Buf, typtbl: *mut *mut u8, t: i32, depth
         idl_trap_with("skip_any: too deeply nested record");
     }
 
+    idl_limit_check(true, 1); // decrement and check quota
+
     if t < 0 {
         // Primitive type
         match t {
@@ -525,9 +534,9 @@ unsafe extern "C" fn skip_fields(tb: *mut Buf, buf: *mut Buf, typtbl: *mut *mut
     }
 }
 
-unsafe fn is_opt_reserved(typtbl: *mut *mut u8, end: *mut u8, t: i32) -> bool {
+unsafe fn is_null_opt_reserved(typtbl: *mut *mut u8, end: *mut u8, t: i32) -> bool {
     if is_primitive_type(false, t) {
-        return t == IDL_PRIM_reserved;
+        return t == IDL_PRIM_null || t == IDL_PRIM_reserved;
     }
 
     // unfold t
@@ -629,7 +638,7 @@ unsafe fn sub(
                 for _ in 0..in1 {
                     let t11 = sleb128_decode(&mut tb1);
                     if in2 == 0 {
-                        if !is_opt_reserved(typtbl1, end1, t11) {
+                        if !is_null_opt_reserved(typtbl1, end1, t11) {
                             break 'return_false;
                         }
                     } else {
@@ -651,7 +660,7 @@ unsafe fn sub(
                 for _ in 0..out2 {
                     let t21 = sleb128_decode(&mut tb2);
                     if out1 == 0 {
-                        if !is_opt_reserved(typtbl2, end2, t21) {
+                        if !is_null_opt_reserved(typtbl2, end2, t21) {
                             break 'return_false;
                         }
                     } else {
@@ -709,7 +718,7 @@ unsafe fn sub(
                     let t21 = sleb128_decode(&mut tb2);
                     if n1 == 0 {
                         // check all remaining fields optional
-                        if !is_opt_reserved(typtbl2, end2, t21) {
+                        if !is_null_opt_reserved(typtbl2, end2, t21) {
                             break 'return_false;
                         }
                         continue;
@@ -725,7 +734,7 @@ unsafe fn sub(
                         }
                     };
                     if tag1 > tag2 {
-                        if !is_opt_reserved(typtbl2, end2, t21) {
+                        if !is_null_opt_reserved(typtbl2, end2, t21) {
                             // missing, non_opt field
                             break 'return_false;
                         }

diff --git a/src/codegen/compile.ml b/src/codegen/compile.ml
@@ -6341,7 +6341,12 @@ module RTS_Exports = struct
     E.add_export env (nr {
       name = Lib.Utf8.decode "moc_stable_mem_set_version";
       edesc = nr (FuncExport (nr moc_stable_mem_set_version_fi))
-    })
+      });
+
+    E.add_export env (nr {
+        name = Lib.Utf8.decode "idl_limit_check";
+        edesc = nr (FuncExport (nr (E.built_in env "idl_limit_check")))
+      })
 
 end (* RTS_Exports *)
 
@@ -6531,13 +6536,23 @@ module MakeSerialization (Strm : Stream) = struct
     G.i (GlobalGet (nr (E.get_global env "__typtbl_idltyps")))
 
   module Registers = struct
+
+    (* interval for checking instruction counter *)
+    let idl_value_numerator = 1l
+    let idl_value_denominator = 1l
+    let idl_value_bias = 1024l
+
     let register_globals env =
       E.add_global32 env "@@rel_buf_opt" Mutable 0l;
       E.add_global32 env "@@data_buf" Mutable 0l;
       E.add_global32 env "@@ref_buf" Mutable 0l;
       E.add_global32 env "@@typtbl" Mutable 0l;
       E.add_global32 env "@@typtbl_end" Mutable 0l;
-      E.add_global32 env "@@typtbl_size" Mutable 0l
+      E.add_global32 env "@@typtbl_size" Mutable 0l;
+      E.add_global32 env "@@value_denominator" Mutable idl_value_denominator;
+      E.add_global32 env "@@value_numerator" Mutable idl_value_numerator;
+      E.add_global32 env "@@value_bias" Mutable idl_value_bias;
+      E.add_global64 env "@@value_quota" Mutable 0L
 
     let get_rel_buf_opt env =
       G.i (GlobalGet (nr (E.get_global env "@@rel_buf_opt")))
@@ -6568,6 +6583,88 @@ module MakeSerialization (Strm : Stream) = struct
       G.i (GlobalGet (nr (E.get_global env "@@typtbl_size")))
     let set_typtbl_size env =
       G.i (GlobalSet (nr (E.get_global env "@@typtbl_size")))
+
+    let get_value_quota env =
+      G.i (GlobalGet (nr (E.get_global env "@@value_quota")))
+    let set_value_quota env =
+      G.i (GlobalSet (nr (E.get_global env "@@value_quota")))
+
+    let get_value_numerator env =
+      G.i (GlobalGet (nr (E.get_global env "@@value_numerator")))
+    let set_value_numerator env =
+      G.i (GlobalSet (nr (E.get_global env "@@value_numerator")))
+
+    let get_value_denominator env =
+      G.i (GlobalGet (nr (E.get_global env "@@value_denominator")))
+    let set_value_denominator env =
+      G.i (GlobalSet (nr (E.get_global env "@@value_denominator")))
+
+    let get_value_bias env =
+      G.i (GlobalGet (nr (E.get_global env "@@value_bias")))
+    let set_value_bias env =
+      G.i (GlobalSet (nr (E.get_global env "@@value_bias")))
+
+    let reset_value_limit env get_blob get_rel_buf_opt =
+      get_rel_buf_opt ^^
+      G.if0
+      begin (* Candid deserialization *)
+        (* Set instruction limit *)
+        (* Use 32-bit factors and terms to (mostly) avoid 64-bit overflow *)
+        let (set_product, get_product) = new_local64 env "product" in
+        get_blob ^^
+        Blob.len env ^^
+        G.i (Convert (Wasm.Values.I64 I64Op.ExtendUI32)) ^^
+        get_value_numerator env ^^
+        G.i (Convert (Wasm.Values.I64 I64Op.ExtendUI32)) ^^
+        G.i (Binary (Wasm.Values.I64 I64Op.Mul)) ^^
+        get_value_denominator env ^^
+        G.i (Convert (Wasm.Values.I64 I64Op.ExtendUI32)) ^^
+        G.i (Binary (Wasm.Values.I64 I64Op.DivU)) ^^
+        set_product ^^
+        get_product ^^
+        get_value_bias env ^^
+        G.i (Convert (Wasm.Values.I64 I64Op.ExtendUI32)) ^^
+        G.i (Binary (Wasm.Values.I64 I64Op.Add)) ^^
+        set_value_quota env ^^
+        (* Saturate value_quota on overflow *)
+        get_value_quota env ^^
+        get_product ^^
+        G.i (Compare (Wasm.Values.I64 I64Op.LtU)) ^^
+        G.if0 begin
+          compile_const_64 (-1L) ^^
+          set_value_quota env
+        end
+          G.nop
+      end
+      begin (* Extended candid/ Destabilization *)
+        G.nop
+      end
+
+    let define_idl_limit_check env =
+      Func.define_built_in env "idl_limit_check"
+        [("decrement", I32Type); ("count", I64Type)] [] (fun env ->
+        get_rel_buf_opt env ^^
+        G.if0 begin (* Candid deserialization *)
+          get_value_quota env ^^
+          G.i (LocalGet (nr 1l)) ^^ (* Count of values *)
+          G.i (Compare (Wasm.Values.I64 I64Op.LtU)) ^^
+          E.then_trap_with env "IDL error: exceeded value limit" ^^
+          (* if (decrement) quota -= count *)
+          G.i (LocalGet (nr 0l)) ^^
+          G.if0 begin
+             get_value_quota env ^^
+             G.i (LocalGet (nr 1l)) ^^
+             G.i (Binary (Wasm.Values.I64 I64Op.Sub)) ^^
+             set_value_quota env
+           end
+             G.nop
+        end begin (* Extended Candid/Destabilization *)
+          G.nop
+        end)
+
+    let idl_limit_check env =
+      G.i (Call (nr (E.built_in env "idl_limit_check")))
+
   end
 
   open Typ_hash
@@ -7224,6 +7321,11 @@ module MakeSerialization (Strm : Stream) = struct
       let get_typtbl_end = Registers.get_typtbl_end env in
       let get_typtbl_size = Registers.get_typtbl_size env in
 
+      (* Decrement and check idl quota *)
+      compile_unboxed_const 1l ^^
+      compile_const_64 1L ^^
+      Registers.idl_limit_check env ^^
+
       (* Check recursion depth (protects against empty record etc.) *)
       (* Factor 2 because at each step, the expected type could go through one
          level of opt that is not present in the value type
@@ -7642,7 +7744,7 @@ module MakeSerialization (Strm : Stream) = struct
               end
               begin
                 match normalize t with
-                | Opt _ | Any -> Opt.null_lit env
+                | Prim Null | Opt _ | Any -> Opt.null_lit env
                 | _ -> coercion_failed "IDL error: did not find tuple field in record"
               end
           ) ts ^^
@@ -7671,7 +7773,7 @@ module MakeSerialization (Strm : Stream) = struct
                   end
                 begin
                   match normalize f.typ with
-                  | Opt _ | Any -> Opt.null_lit env
+                  | Prim Null | Opt _ | Any -> Opt.null_lit env
                   | _ -> coercion_failed (Printf.sprintf "IDL error: did not find field %s in record" f.lab)
                 end
           ) (sort_by_hash fs)) ^^
@@ -7733,6 +7835,10 @@ module MakeSerialization (Strm : Stream) = struct
           ReadBuf.read_sleb128 env get_typ_buf ^^
           set_arg_typ ^^
           ReadBuf.read_leb128 env get_data_buf ^^ set_len ^^
+          (* Don't decrement just check quota *)
+          compile_unboxed_const 0l ^^
+          get_len ^^ G.i (Convert (Wasm.Values.I64 I64Op.ExtendUI32)) ^^
+          Registers.idl_limit_check env ^^
           get_len ^^ Arr.alloc env ^^ set_x ^^
           get_len ^^ from_0_to_n env (fun get_i ->
           get_x ^^ get_i ^^ Arr.unsafe_idx env ^^
@@ -7978,6 +8084,16 @@ module MakeSerialization (Strm : Stream) = struct
 
       (* Allocate memo table, if necessary *)
       with_rel_buf_opt env extended (get_typtbl_size_ptr ^^ load_unskewed_ptr) (fun get_rel_buf_opt ->
+      begin
+        (* set up invariant register arguments *)
+        get_rel_buf_opt ^^ Registers.set_rel_buf_opt env ^^
+        get_data_buf ^^ Registers.set_data_buf env ^^
+        get_ref_buf ^^ Registers.set_ref_buf env ^^
+        get_typtbl_ptr ^^ load_unskewed_ptr ^^ Registers.set_typtbl env ^^
+        get_maintyps_ptr ^^ load_unskewed_ptr ^^ Registers.set_typtbl_end env ^^
+        get_typtbl_size_ptr ^^ load_unskewed_ptr ^^ Registers.set_typtbl_size env ^^
+        Registers.reset_value_limit env get_blob get_rel_buf_opt
+      end ^^
 
       (* set up a dedicated read buffer for the list of main types *)
       ReadBuf.alloc env (fun get_main_typs_buf ->
@@ -7988,7 +8104,7 @@ module MakeSerialization (Strm : Stream) = struct
         G.concat_map (fun t ->
           let can_recover, default_or_trap = Type.(
             match normalize t with
-            | Opt _ | Any ->
+            | Prim Null | Opt _ | Any ->
               (Bool.lit true, fun msg -> Opt.null_lit env)
             | _ ->
               (get_can_recover, fun msg ->
@@ -8002,15 +8118,6 @@ module MakeSerialization (Strm : Stream) = struct
           G.if1 I32Type
            (default_or_trap ("IDL error: too few arguments " ^ ts_name))
            (begin
-              begin
-                (* set up invariant register arguments *)
-                get_rel_buf_opt ^^ Registers.set_rel_buf_opt env ^^
-                get_data_buf ^^ Registers.set_data_buf env ^^
-                get_ref_buf ^^ Registers.set_ref_buf env ^^
-                get_typtbl_ptr ^^ load_unskewed_ptr ^^ Registers.set_typtbl env ^^
-                get_maintyps_ptr ^^ load_unskewed_ptr ^^ Registers.set_typtbl_end env ^^
-                get_typtbl_size_ptr ^^ load_unskewed_ptr ^^ Registers.set_typtbl_size env
-              end ^^
               (* set up variable frame arguments *)
               Stack.with_frame env "frame_ptr" 3l (fun () ->
                 (* idltyp *)
@@ -11735,6 +11842,26 @@ and compile_prim_invocation (env : E.t) ae p es at =
   | OtherPrim "btstInt64", [_;_] ->
     const_sr (SR.UnboxedWord64 Type.Int64) (Word64.btst_kernel env)
 
+  | OtherPrim "setCandidLimits", [e1; e2; e3] ->
+    SR.unit,
+    compile_exp_as env ae (SR.UnboxedWord32 Type.Nat32) e1 ^^
+    Serialization.Registers.set_value_numerator env ^^
+    compile_exp_as env ae (SR.UnboxedWord32 Type.Nat32) e2 ^^
+    Serialization.Registers.set_value_denominator env ^^
+    Serialization.Registers.get_value_denominator env ^^
+    E.else_trap_with env "Candid limit denominator cannot be zero" ^^
+    compile_exp_as env ae (SR.UnboxedWord32 Type.Nat32) e3 ^^
+    Serialization.Registers.set_value_bias env
+
+  | OtherPrim "getCandidLimits", [] ->
+    SR.UnboxedTuple 3,
+    Serialization.Registers.get_value_numerator env ^^
+    BoxedSmallWord.box env Type.Nat32 ^^
+    Serialization.Registers.get_value_denominator env ^^
+    BoxedSmallWord.box env Type.Nat32 ^^
+    Serialization.Registers.get_value_bias env ^^
+    BoxedSmallWord.box env Type.Nat32
+
   (* Coercions for abstract types *)
   | CastPrim (_,_), [e] ->
     compile_exp env ae e
@@ -12823,6 +12950,7 @@ let compile mode rts (prog : Ir.prog) : Wasm_exts.CustomModule.extended_module =
   GC.register_globals env;
   StableMem.register_globals env;
   Serialization.Registers.register_globals env;
+  Serialization.Registers.define_idl_limit_check env;
 
   (* See Note [Candid subtype checks] *)
   let set_serialization_globals = Serialization.register_delayed_globals env in

diff --git a/src/mo_values/prim.ml b/src/mo_values/prim.ml
@@ -382,4 +382,11 @@ let prim trap =
   | "canister_version" ->
       fun _ v k -> as_unit v; k (Nat64 (Numerics.Nat64.of_int 42))
 
+  (* fake *)
+  | "setCandidLimits" ->
+      fun _ v k -> k unit
+  | "getCandidLimits" ->
+      fun _ v k -> k (Tup [
+        Nat32 Numerics.Nat32.zero; Nat32 Numerics.Nat32.zero; Nat32 Numerics.Nat32.zero])
+
   | s -> trap.trap ("Value.prim: " ^ s)