From 278a52c35c55367db8c0e6fafa853c49564107e2 Mon Sep 17 00:00:00 2001 From: kojix2 <2xijok@gmail.com> Date: Thu, 16 May 2024 13:19:22 +0900 Subject: [PATCH] Ensure allowed_special is not null --- src/lib.rs | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/src/lib.rs b/src/lib.rs index 154a04f..bfa9d48 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -799,7 +799,20 @@ mod tests { let text = CString::new("I am a cat. <|endoftext|>").unwrap(); let mut num_tokens: usize = 0; let corebpe = c_get_bpe_from_model(model.as_ptr()); - let tokens = c_corebpe_encode(corebpe, text.as_ptr(), std::ptr::null(), 0, &mut num_tokens); + + // zero-length slices require a non-null pointer + // Create a CString and an array of pointers to pass to the function + let placeholder = CString::new("").unwrap(); + let placeholder_ptr: *const i8 = placeholder.as_ptr(); + let ptr_array: [*const i8; 1] = [placeholder_ptr]; + + let tokens = c_corebpe_encode( + corebpe, + text.as_ptr(), + ptr_array.as_ptr(), + 0, + &mut num_tokens, + ); assert_eq!(num_tokens, 11); let tokens = unsafe { std::slice::from_raw_parts(tokens, num_tokens) }; let tokens: Vec = tokens.iter().map(|&x| x as usize).collect();