fix(gguf): 移除不必要的 tokenizer.pre 要求

Signed-off-by: YdrMaster <ydrml@hotmail.com>
InfiniTensor · Jan 25, 2025 · e078839 · e078839
1 parent c391886
commit e078839
Show file tree

Hide file tree

Showing 2 changed files with 1 addition and 2 deletions.
diff --git a/Cargo.toml b/Cargo.toml
@@ -34,7 +34,7 @@ itertools = "0.13"
 env_logger = "0.11"
 build-script-cfg = "0.0"
 
-operators = { git = "https://github.com/YdrMaster/operators-rs", rev = "c81600f", default-features = false }
+operators = { git = "https://github.com/YdrMaster/operators-rs", rev = "d88d716", default-features = false }
 
 search-cl-tools = { git = "https://github.com/InfiniTensor/clrt", rev = "9b6289d" }
 search-infini-tools = { git = "https://github.com/InfiniTensor/infini-rt", rev = "f40bcb5" }

diff --git a/gguf/src/tokenizer.rs b/gguf/src/tokenizer.rs
@@ -61,7 +61,6 @@ impl Tokenizer {
     }
 
     fn bpe_from_gguf(gguf: &GGufModel) -> Self {
-        let _pre = gguf.get_str("tokenizer.ggml.pre").unwrap();
         let tokens = gguf.tokenizer_ggml_tokens().unwrap();
         let scores = gguf.tokenizer_ggml_scores().unwrap();
         let token_type = gguf.tokenizer_ggml_token_type().unwrap();