Ensure allowed_special is not null

src/lib.rs

@@ -799,7 +799,20 @@ mod tests {
         let text = CString::new("I am a cat. <|endoftext|>").unwrap();
         let mut num_tokens: usize = 0;
         let corebpe = c_get_bpe_from_model(model.as_ptr());
-        let tokens = c_corebpe_encode(corebpe, text.as_ptr(), std::ptr::null(), 0, &mut num_tokens);
+
+        // zero-length slices require a non-null pointer
+        // Create a CString and an array of pointers to pass to the function
+        let placeholder = CString::new("").unwrap();
+        let placeholder_ptr: *const i8 = placeholder.as_ptr();
+        let ptr_array: [*const i8; 1] = [placeholder_ptr];
+
+        let tokens = c_corebpe_encode(
+            corebpe,
+            text.as_ptr(),
+            ptr_array.as_ptr(),
+            0,
+            &mut num_tokens,
+        );
         assert_eq!(num_tokens, 11);
         let tokens = unsafe { std::slice::from_raw_parts(tokens, num_tokens) };
         let tokens: Vec<usize> = tokens.iter().map(|&x| x as usize).collect();