Skip to content

Commit

Permalink
experiment: value quota based idl decoder limiting (#4657)
Browse files Browse the repository at this point in the history 
Simplifies #4624 to a simple linear limit on the number of decoded values as a function of decoded payload size,
instead of using two linear functions on perfcounter (simulated or real) and allocation counter.

The function is:

value_quota(blob) : Nat64 = blob.size() * (numerator/denominator) + bias

where blob is the candid blob to be decoded, and `numerator` (default 1), `denominator` (default 1) and `bias` (default 1024) are `Nat32s`.


Much simpler than #4624 and doesn't depend on vagaries of instruction metering and byte allocation which varies with gc and compiler options, but is it good enough?

The constants can be (globally) modified/inspected using prims (Prim.getCandidLimits/Prim.setCandidLimits) which will need to get exposed in base eventually.

The quota is decremented on every call to deserialise or skip a value in vanilla candid mode (destabilization is not metered).
The quota is eagerly checked before deserializing or skipping arrays.

One possible refinement would be to combine the value quota with a memory quota (though the latter would still vary with gc flavour and perhaps word-size unless we count logical words)


- [x] Disable for destabilization (iff Registers.get_rel_buf_opt is zero)
- [x] Port new candid spacebomb test suite to drun-tests, to test against real perf counter provided by drun. 
- [x] Bump candid dependency to most recent
- [x] Pass new spacebomb tests, both in candid test suite on wasmtime using value counter.
  • Loading branch information
@crusso
crusso authored Aug 16, 2024
1 parent 367145d commit b03a4d6
Show file tree
Hide file tree
Showing 26 changed files with 581 additions and 51 deletions.
16 changes: 16 additions & 0 deletions Changelog.md
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,21 @@
# Motoko compiler changelog

* motoko (`moc`)

* Candid decoding: impose an upper limit on the number of values decoded or skipped in a single candid payload,
as a linear function, `max_values`, of binary payload size.

```
max_values(blob) = (blob.size() * numerator)/denominator + bias
```
The current default settings are `{numerator = 1; denominator = 1; bias = 1024 }`, allowing a maximum
of 1024 values plus one additional value per byte in the payload.
While hopefully not required, the constant factors can be read/modified using system functions:
* Prim.setCandidLimits: `<system>{numerator : Nat32; denominator : Nat32; bias : Nat32 } -> ()`
* Prim.getCandidLimits: `<system>() -> {numerator : Nat32; denominator : Nat32; bias : Nat32 }`
## 0.12.1 (2024-08-08)
* motoko (`moc`)
Expand Down
6 changes: 3 additions & 3 deletions nix/sources.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,10 +6,10 @@
"homepage": "",
"owner": "dfinity",
"repo": "candid",
"rev": "331217bae379cbebfa531a140f2186c99fae1425",
"sha256": "095w2a4lxy2gd7vfjxn7jszm4x3srw8xlxb1zzd096y6h047rxlj",
"rev": "34b4eb0b581bbf04902e20bf1370e3a293d1956f",
"sha256": "1gr49p938hzm8fq4r3n7j2lzfj0hmah5sb411sma24plnmwy7ljx",
"type": "tarball",
"url": "https://github.com/dfinity/candid/archive/331217bae379cbebfa531a140f2186c99fae1425.tar.gz",
"url": "https://github.com/dfinity/candid/archive/34b4eb0b581bbf04902e20bf1370e3a293d1956f.tar.gz",
"url_template": "https://github.com/<owner>/<repo>/archive/<rev>.tar.gz"
},
"esm": {
Expand Down
21 changes: 15 additions & 6 deletions rts/motoko-rts/src/idl.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,11 @@ use core::cmp::min;

use motoko_rts_macros::ic_mem_fn;

extern "C" {
// check instruction decoding limit, exported by moc
pub fn idl_limit_check(decrement: bool, value_count: u64);
}

//
// IDL constants
//
Expand Down Expand Up @@ -306,6 +311,7 @@ unsafe fn skip_any_vec(buf: *mut Buf, typtbl: *mut *mut u8, t: i32, count: u32)
if count == 0 {
return;
}
idl_limit_check(false, count as u64);
let ptr_before = (*buf).ptr;
skip_any(buf, typtbl, t, 0);
let ptr_after = (*buf).ptr;
Expand All @@ -314,6 +320,7 @@ unsafe fn skip_any_vec(buf: *mut Buf, typtbl: *mut *mut u8, t: i32, count: u32)
// makes no progress. No point in calling it over and over again.
// (This is easier to detect this way than by analyzing the type table,
// where we’d have to chase single-field-records.)
idl_limit_check(true, (count - 1) as u64);
return;
}
for _ in 1..count {
Expand All @@ -332,6 +339,8 @@ unsafe extern "C" fn skip_any(buf: *mut Buf, typtbl: *mut *mut u8, t: i32, depth
idl_trap_with("skip_any: too deeply nested record");
}

idl_limit_check(true, 1); // decrement and check quota

if t < 0 {
// Primitive type
match t {
Expand Down Expand Up @@ -525,9 +534,9 @@ unsafe extern "C" fn skip_fields(tb: *mut Buf, buf: *mut Buf, typtbl: *mut *mut
}
}

unsafe fn is_opt_reserved(typtbl: *mut *mut u8, end: *mut u8, t: i32) -> bool {
unsafe fn is_null_opt_reserved(typtbl: *mut *mut u8, end: *mut u8, t: i32) -> bool {
if is_primitive_type(false, t) {
return t == IDL_PRIM_reserved;
return t == IDL_PRIM_null || t == IDL_PRIM_reserved;
}

// unfold t
Expand Down Expand Up @@ -629,7 +638,7 @@ unsafe fn sub(
for _ in 0..in1 {
let t11 = sleb128_decode(&mut tb1);
if in2 == 0 {
if !is_opt_reserved(typtbl1, end1, t11) {
if !is_null_opt_reserved(typtbl1, end1, t11) {
break 'return_false;
}
} else {
Expand All @@ -651,7 +660,7 @@ unsafe fn sub(
for _ in 0..out2 {
let t21 = sleb128_decode(&mut tb2);
if out1 == 0 {
if !is_opt_reserved(typtbl2, end2, t21) {
if !is_null_opt_reserved(typtbl2, end2, t21) {
break 'return_false;
}
} else {
Expand Down Expand Up @@ -709,7 +718,7 @@ unsafe fn sub(
let t21 = sleb128_decode(&mut tb2);
if n1 == 0 {
// check all remaining fields optional
if !is_opt_reserved(typtbl2, end2, t21) {
if !is_null_opt_reserved(typtbl2, end2, t21) {
break 'return_false;
}
continue;
Expand All @@ -725,7 +734,7 @@ unsafe fn sub(
}
};
if tag1 > tag2 {
if !is_opt_reserved(typtbl2, end2, t21) {
if !is_null_opt_reserved(typtbl2, end2, t21) {
// missing, non_opt field
break 'return_false;
}
Expand Down
156 changes: 142 additions & 14 deletions src/codegen/compile.ml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6341,7 +6341,12 @@ module RTS_Exports = struct
E.add_export env (nr {
name = Lib.Utf8.decode "moc_stable_mem_set_version";
edesc = nr (FuncExport (nr moc_stable_mem_set_version_fi))
})
});

E.add_export env (nr {
name = Lib.Utf8.decode "idl_limit_check";
edesc = nr (FuncExport (nr (E.built_in env "idl_limit_check")))
})

end (* RTS_Exports *)

Expand Down Expand Up @@ -6531,13 +6536,23 @@ module MakeSerialization (Strm : Stream) = struct
G.i (GlobalGet (nr (E.get_global env "__typtbl_idltyps")))

module Registers = struct

(* interval for checking instruction counter *)
let idl_value_numerator = 1l
let idl_value_denominator = 1l
let idl_value_bias = 1024l

let register_globals env =
E.add_global32 env "@@rel_buf_opt" Mutable 0l;
E.add_global32 env "@@data_buf" Mutable 0l;
E.add_global32 env "@@ref_buf" Mutable 0l;
E.add_global32 env "@@typtbl" Mutable 0l;
E.add_global32 env "@@typtbl_end" Mutable 0l;
E.add_global32 env "@@typtbl_size" Mutable 0l
E.add_global32 env "@@typtbl_size" Mutable 0l;
E.add_global32 env "@@value_denominator" Mutable idl_value_denominator;
E.add_global32 env "@@value_numerator" Mutable idl_value_numerator;
E.add_global32 env "@@value_bias" Mutable idl_value_bias;
E.add_global64 env "@@value_quota" Mutable 0L

let get_rel_buf_opt env =
G.i (GlobalGet (nr (E.get_global env "@@rel_buf_opt")))
Expand Down Expand Up @@ -6568,6 +6583,88 @@ module MakeSerialization (Strm : Stream) = struct
G.i (GlobalGet (nr (E.get_global env "@@typtbl_size")))
let set_typtbl_size env =
G.i (GlobalSet (nr (E.get_global env "@@typtbl_size")))

let get_value_quota env =
G.i (GlobalGet (nr (E.get_global env "@@value_quota")))
let set_value_quota env =
G.i (GlobalSet (nr (E.get_global env "@@value_quota")))

let get_value_numerator env =
G.i (GlobalGet (nr (E.get_global env "@@value_numerator")))
let set_value_numerator env =
G.i (GlobalSet (nr (E.get_global env "@@value_numerator")))

let get_value_denominator env =
G.i (GlobalGet (nr (E.get_global env "@@value_denominator")))
let set_value_denominator env =
G.i (GlobalSet (nr (E.get_global env "@@value_denominator")))

let get_value_bias env =
G.i (GlobalGet (nr (E.get_global env "@@value_bias")))
let set_value_bias env =
G.i (GlobalSet (nr (E.get_global env "@@value_bias")))

let reset_value_limit env get_blob get_rel_buf_opt =
get_rel_buf_opt ^^
G.if0
begin (* Candid deserialization *)
(* Set instruction limit *)
(* Use 32-bit factors and terms to (mostly) avoid 64-bit overflow *)
let (set_product, get_product) = new_local64 env "product" in
get_blob ^^
Blob.len env ^^
G.i (Convert (Wasm.Values.I64 I64Op.ExtendUI32)) ^^
get_value_numerator env ^^
G.i (Convert (Wasm.Values.I64 I64Op.ExtendUI32)) ^^
G.i (Binary (Wasm.Values.I64 I64Op.Mul)) ^^
get_value_denominator env ^^
G.i (Convert (Wasm.Values.I64 I64Op.ExtendUI32)) ^^
G.i (Binary (Wasm.Values.I64 I64Op.DivU)) ^^
set_product ^^
get_product ^^
get_value_bias env ^^
G.i (Convert (Wasm.Values.I64 I64Op.ExtendUI32)) ^^
G.i (Binary (Wasm.Values.I64 I64Op.Add)) ^^
set_value_quota env ^^
(* Saturate value_quota on overflow *)
get_value_quota env ^^
get_product ^^
G.i (Compare (Wasm.Values.I64 I64Op.LtU)) ^^
G.if0 begin
compile_const_64 (-1L) ^^
set_value_quota env
end
G.nop
end
begin (* Extended candid/ Destabilization *)
G.nop
end

let define_idl_limit_check env =
Func.define_built_in env "idl_limit_check"
[("decrement", I32Type); ("count", I64Type)] [] (fun env ->
get_rel_buf_opt env ^^
G.if0 begin (* Candid deserialization *)
get_value_quota env ^^
G.i (LocalGet (nr 1l)) ^^ (* Count of values *)
G.i (Compare (Wasm.Values.I64 I64Op.LtU)) ^^
E.then_trap_with env "IDL error: exceeded value limit" ^^
(* if (decrement) quota -= count *)
G.i (LocalGet (nr 0l)) ^^
G.if0 begin
get_value_quota env ^^
G.i (LocalGet (nr 1l)) ^^
G.i (Binary (Wasm.Values.I64 I64Op.Sub)) ^^
set_value_quota env
end
G.nop
end begin (* Extended Candid/Destabilization *)
G.nop
end)

let idl_limit_check env =
G.i (Call (nr (E.built_in env "idl_limit_check")))

end

open Typ_hash
Expand Down Expand Up @@ -7224,6 +7321,11 @@ module MakeSerialization (Strm : Stream) = struct
let get_typtbl_end = Registers.get_typtbl_end env in
let get_typtbl_size = Registers.get_typtbl_size env in

(* Decrement and check idl quota *)
compile_unboxed_const 1l ^^
compile_const_64 1L ^^
Registers.idl_limit_check env ^^

(* Check recursion depth (protects against empty record etc.) *)
(* Factor 2 because at each step, the expected type could go through one
level of opt that is not present in the value type
Expand Down Expand Up @@ -7642,7 +7744,7 @@ module MakeSerialization (Strm : Stream) = struct
end
begin
match normalize t with
| Opt _ | Any -> Opt.null_lit env
| Prim Null | Opt _ | Any -> Opt.null_lit env
| _ -> coercion_failed "IDL error: did not find tuple field in record"
end
) ts ^^
Expand Down Expand Up @@ -7671,7 +7773,7 @@ module MakeSerialization (Strm : Stream) = struct
end
begin
match normalize f.typ with
| Opt _ | Any -> Opt.null_lit env
| Prim Null | Opt _ | Any -> Opt.null_lit env
| _ -> coercion_failed (Printf.sprintf "IDL error: did not find field %s in record" f.lab)
end
) (sort_by_hash fs)) ^^
Expand Down Expand Up @@ -7733,6 +7835,10 @@ module MakeSerialization (Strm : Stream) = struct
ReadBuf.read_sleb128 env get_typ_buf ^^
set_arg_typ ^^
ReadBuf.read_leb128 env get_data_buf ^^ set_len ^^
(* Don't decrement just check quota *)
compile_unboxed_const 0l ^^
get_len ^^ G.i (Convert (Wasm.Values.I64 I64Op.ExtendUI32)) ^^
Registers.idl_limit_check env ^^
get_len ^^ Arr.alloc env ^^ set_x ^^
get_len ^^ from_0_to_n env (fun get_i ->
get_x ^^ get_i ^^ Arr.unsafe_idx env ^^
Expand Down Expand Up @@ -7978,6 +8084,16 @@ module MakeSerialization (Strm : Stream) = struct

(* Allocate memo table, if necessary *)
with_rel_buf_opt env extended (get_typtbl_size_ptr ^^ load_unskewed_ptr) (fun get_rel_buf_opt ->
begin
(* set up invariant register arguments *)
get_rel_buf_opt ^^ Registers.set_rel_buf_opt env ^^
get_data_buf ^^ Registers.set_data_buf env ^^
get_ref_buf ^^ Registers.set_ref_buf env ^^
get_typtbl_ptr ^^ load_unskewed_ptr ^^ Registers.set_typtbl env ^^
get_maintyps_ptr ^^ load_unskewed_ptr ^^ Registers.set_typtbl_end env ^^
get_typtbl_size_ptr ^^ load_unskewed_ptr ^^ Registers.set_typtbl_size env ^^
Registers.reset_value_limit env get_blob get_rel_buf_opt
end ^^

(* set up a dedicated read buffer for the list of main types *)
ReadBuf.alloc env (fun get_main_typs_buf ->
Expand All @@ -7988,7 +8104,7 @@ module MakeSerialization (Strm : Stream) = struct
G.concat_map (fun t ->
let can_recover, default_or_trap = Type.(
match normalize t with
| Opt _ | Any ->
| Prim Null | Opt _ | Any ->
(Bool.lit true, fun msg -> Opt.null_lit env)
| _ ->
(get_can_recover, fun msg ->
Expand All @@ -8002,15 +8118,6 @@ module MakeSerialization (Strm : Stream) = struct
G.if1 I32Type
(default_or_trap ("IDL error: too few arguments " ^ ts_name))
(begin
begin
(* set up invariant register arguments *)
get_rel_buf_opt ^^ Registers.set_rel_buf_opt env ^^
get_data_buf ^^ Registers.set_data_buf env ^^
get_ref_buf ^^ Registers.set_ref_buf env ^^
get_typtbl_ptr ^^ load_unskewed_ptr ^^ Registers.set_typtbl env ^^
get_maintyps_ptr ^^ load_unskewed_ptr ^^ Registers.set_typtbl_end env ^^
get_typtbl_size_ptr ^^ load_unskewed_ptr ^^ Registers.set_typtbl_size env
end ^^
(* set up variable frame arguments *)
Stack.with_frame env "frame_ptr" 3l (fun () ->
(* idltyp *)
Expand Down Expand Up @@ -11735,6 +11842,26 @@ and compile_prim_invocation (env : E.t) ae p es at =
| OtherPrim "btstInt64", [_;_] ->
const_sr (SR.UnboxedWord64 Type.Int64) (Word64.btst_kernel env)

| OtherPrim "setCandidLimits", [e1; e2; e3] ->
SR.unit,
compile_exp_as env ae (SR.UnboxedWord32 Type.Nat32) e1 ^^
Serialization.Registers.set_value_numerator env ^^
compile_exp_as env ae (SR.UnboxedWord32 Type.Nat32) e2 ^^
Serialization.Registers.set_value_denominator env ^^
Serialization.Registers.get_value_denominator env ^^
E.else_trap_with env "Candid limit denominator cannot be zero" ^^
compile_exp_as env ae (SR.UnboxedWord32 Type.Nat32) e3 ^^
Serialization.Registers.set_value_bias env

| OtherPrim "getCandidLimits", [] ->
SR.UnboxedTuple 3,
Serialization.Registers.get_value_numerator env ^^
BoxedSmallWord.box env Type.Nat32 ^^
Serialization.Registers.get_value_denominator env ^^
BoxedSmallWord.box env Type.Nat32 ^^
Serialization.Registers.get_value_bias env ^^
BoxedSmallWord.box env Type.Nat32

(* Coercions for abstract types *)
| CastPrim (_,_), [e] ->
compile_exp env ae e
Expand Down Expand Up @@ -12823,6 +12950,7 @@ let compile mode rts (prog : Ir.prog) : Wasm_exts.CustomModule.extended_module =
GC.register_globals env;
StableMem.register_globals env;
Serialization.Registers.register_globals env;
Serialization.Registers.define_idl_limit_check env;

(* See Note [Candid subtype checks] *)
let set_serialization_globals = Serialization.register_delayed_globals env in
Expand Down
7 changes: 7 additions & 0 deletions src/mo_values/prim.ml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -382,4 +382,11 @@ let prim trap =
| "canister_version" ->
fun _ v k -> as_unit v; k (Nat64 (Numerics.Nat64.of_int 42))

(* fake *)
| "setCandidLimits" ->
fun _ v k -> k unit
| "getCandidLimits" ->
fun _ v k -> k (Tup [
Nat32 Numerics.Nat32.zero; Nat32 Numerics.Nat32.zero; Nat32 Numerics.Nat32.zero])

| s -> trap.trap ("Value.prim: " ^ s)
Loading

0 comments on commit b03a4d6

Please sign in to comment.