feat(clip): 添加 clip-cpu

Signed-off-by: YdrMaster <ydrml@hotmail.com>

Cargo.toml

@@ -12,6 +12,7 @@ members = [
     "models/llama/ascend",
 
     "models/clip/common",
+    "models/clip/common-cpu",
 ]
 resolver = "2"
 

models/clip/common-cpu/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "clip-cpu"
+version = "0.0.0"
+edition = "2021"
+authors = ["YdrMaster <ydrml@hotmail.com>"]
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+clip.path = "../common"
+operators = { workspace = true, features = ["common-cpu"] }
+
+[dev-dependencies]
+test-utils.workspace = true
+gguf.workspace = true

models/clip/common-cpu/src/lib.rs

@@ -0,0 +1,2 @@
+#[cfg(test)]
+mod test_infer;

models/clip/common-cpu/src/test_infer.rs

@@ -0,0 +1,40 @@
+use clip::{ClipMeta, ClipStorage, Image};
+use gguf::GGufModel;
+use std::time::Instant;
+use test_utils::Inference;
+
+#[test]
+fn test_infer() {
+    let Some(Inference { model, .. }) = Inference::load() else {
+        return;
+    };
+    let Some(picture) = test_utils::image() else {
+        return;
+    };
+
+    let gguf = GGufModel::read(model.iter().map(|s| &**s));
+    let storage = ClipStorage::from_gguf(&gguf);
+    let meta = &storage.meta;
+    println!("{meta:#?}");
+
+    let &ClipMeta {
+        dt_embd,
+
+        d_image,
+        d_patch,
+
+        image_mean,
+        image_std,
+        ..
+    } = meta;
+
+    let time = Instant::now();
+    let image = Image::load(picture);
+    println!("load image {:?}", time.elapsed());
+
+    let time = Instant::now();
+    let _slices = image
+        .slice_uhd(9, d_image, d_patch)
+        .normalize(dt_embd, image_mean, image_std);
+    println!("slice image {:?}", time.elapsed());
+}

models/clip/common/src/image.rs

@@ -160,7 +160,6 @@ where
     }
 
     /// NHWC rgb Tensor -> NCHW value Tensor
-    #[inline]
     pub fn to_nchw(&self) -> Tensor<&[u8]> {
         self.0
             .destruct_array()
@@ -176,6 +175,7 @@ impl ImageGrid {
         &self.whole
     }
 
+    #[inline]
     pub fn grid(&self) -> [usize; 2] {
         if let Some(grid) = &self.grid {
             let &[y, x, _, _] = grid.shape() else {

models/clip/common/src/lib.rs

@@ -1,9 +1,11 @@
 mod image;
+mod storage;
 
 use gguf::ggml_quants::digit_layout::DigitLayout;
 use tensor::Tensor;
 
 pub use image::{Image, ImageGrid};
+pub use storage::Storage as ClipStorage;
 
 #[derive(Clone, Debug)]
 pub struct ClipMeta {
@@ -21,6 +23,8 @@ pub struct ClipMeta {
     pub d: usize,
     pub di: usize,
 
+    pub image_mean: [f32; 3],
+    pub image_std: [f32; 3],
     pub epsilon: f32,
 }
 
@@ -63,7 +67,13 @@ impl ClipMeta {
         }
     }
 
-    pub fn embd(&self) -> Tensor<usize> {
-        Tensor::new(self.dt_embd, &[self.n_patch(), self.n_mmproj_embd()])
+    pub fn patch_embd(&self) -> Tensor<usize> {
+        let &Self { d, d_patch, .. } = self;
+        Tensor::new(self.dt_mat, &[d, 3, d_patch, d_patch])
+    }
+
+    pub fn patch_embd_bias(&self) -> Tensor<usize> {
+        let &Self { d, .. } = self;
+        Tensor::new(self.dt_bias, &[d])
     }
 }

models/clip/common/src/storage.rs

@@ -1,18 +1,18 @@
 use crate::{ClipMeta, ProjectorType};
 use gguf::{GGufMetaMapExt, GGufModel};
-use std::marker::PhantomData;
 
 #[derive(Clone)]
 pub struct Storage<T> {
     pub meta: ClipMeta,
-    _phantom: PhantomData<T>,
+    pub patch_embd_w: T,
+    pub patch_embd_b: T,
 }
 
 impl<'a> Storage<&'a [u8]> {
     pub fn from_gguf(gguf: &GGufModel<'a>) -> Self {
         let position_embd = &gguf.tensors["v.position_embd.weight"];
-        let w_patch_embd = &gguf.tensors["v.patch_embd.weight"];
-        let b_patch_embd = &gguf.tensors["v.patch_embd.bias"];
+        let patch_embd_w = &gguf.tensors["v.patch_embd.weight"];
+        let patch_embd_b = &gguf.tensors["v.patch_embd.bias"];
 
         let projector = match gguf.get_str("clip.projector_type").unwrap() {
             "mlp" => ProjectorType::Mlp,
@@ -28,21 +28,45 @@ impl<'a> Storage<&'a [u8]> {
             minicpmv_version: gguf.get_usize("clip.minicpmv_version").unwrap() as _,
 
             dt_embd: position_embd.ty,
-            dt_mat :  w_patch_embd.ty,
-            dt_bias:  b_patch_embd.ty,
+            dt_mat :  patch_embd_w.ty,
+            dt_bias:  patch_embd_b.ty,
 
             nblk   : gguf.get_usize("clip.vision.block_count"                 ).unwrap(),
             d_patch: gguf.get_usize("clip.vision.patch_size"                  ).unwrap(),
             d_image: gguf.get_usize("clip.vision.image_size"                  ).unwrap(),
             nh     : gguf.get_usize("clip.vision.attention.head_count"        ).unwrap(),
             d      : gguf.get_usize("clip.vision.embedding_length"            ).unwrap(),
             di     : gguf.get_usize("clip.vision.feed_forward_length"         ).unwrap(),
-            epsilon: gguf.get_f32  ("clip.vision.attention.layer_norm_epsilon").unwrap(),
+
+            image_mean: get_rgb(gguf, "clip.vision.image_mean"),
+            image_std : get_rgb(gguf, "clip.vision.image_std" ),
+            epsilon   : gguf.get_f32("clip.vision.attention.layer_norm_epsilon").unwrap(),
         };
 
         Self {
             meta,
-            _phantom: PhantomData,
+            patch_embd_w: patch_embd_w.data,
+            patch_embd_b: patch_embd_b.data,
         }
     }
 }
+
+fn get_rgb(gguf: &GGufModel, key: &str) -> [f32; 3] {
+    let mut arr = gguf.get_f32_arr(key).unwrap();
+    let mut ans = [0.0; 3];
+    for x in ans.iter_mut() {
+        *x = arr.next().unwrap().unwrap();
+    }
+    ans
+}
+
+#[test]
+fn test() {
+    use test_utils::Inference;
+    let Some(Inference { model, .. }) = Inference::load() else {
+        return;
+    };
+    let gguf = GGufModel::read(model.iter().map(|s| &**s));
+    let storage = Storage::from_gguf(&gguf);
+    println!("{:#?}", storage.meta);
+}

models/llama/common/src/storage.rs

@@ -215,12 +215,12 @@ impl<'w> BlkStorage<&'w [u8]> {
 }
 
 #[test]
-fn test_load() {
+fn test() {
     use test_utils::Inference;
     let Some(Inference { model, .. }) = Inference::load() else {
         return;
     };
     let gguf = GGufModel::read(model.iter().map(|s| &**s));
-    let llama = Storage::from_gguf(&gguf);
-    println!("{:?}", llama.meta);
+    let storage = Storage::from_gguf(&gguf);
+    println!("{:#?}", storage.meta);
 }