support fetch image api

README.md

@@ -10,9 +10,8 @@ Online instance: [ufish-web](https://ufish-team.github.io/#/)
 Usage example:
 
 ```Python
-import numpy as np
-ufish = await api.createWindow("https://ufish-team.github.io/")
-img = np.random.rand(100, 100)  # generate a random image
+ufish = await api.createWindow("http://localhost:5173/")
+img = await ufish.fetchImage("https://huggingface.co/datasets/NaNg/TestData/resolve/main/FISH_spots/MERFISH_1.tif")
 out = await ufish.predict(img)  # predict the spots
 print(out.enhanced.shape)
 print(out.spots.shape)
@@ -29,6 +28,7 @@ CLick to run it using: [web-python-console](https://nanguage.github.io/web-pytho
 | `setInputImage` | `img: np.ndarray; name: string` | Set the input image. |
 | `getOutput` | | Get the output of the model. It's an object contain `enhanced: np.ndarray` and `spots: np.ndarray` fields |
 | `runPredict` | | Run the prediction. |
+| `fetchImage` | `url: string` | Fetch an image from the given url, return a numpy array |
 
 ## Development
 

public/plugins/ufish.imjoy.html

@@ -123,8 +123,7 @@
         self.enhanced_image = None  # enhanced image
         self.spots = None  # spots
 
-    @staticmethod
-    def load_image_from_bytes(file_name, img_bytes):
+    async def load_image_from_bytes(self, file_name, img_bytes):
         _file = io.BytesIO(img_bytes)
         _file.name = file_name
         if file_name.endswith(".tif") or file_name.endswith(".tiff"):
@@ -139,15 +138,14 @@
             image = imageio.imread(_file)
         return image
 
-    @staticmethod
-    def save_image_to_bytes(image):
+    async def save_image_to_bytes(self, image):
         _file = io.BytesIO()
         with tifffile.TiffWriter(_file) as tif:
             tif.save(image)
         return _file.getvalue()
 
     async def view_img_from_bytes(self, file_name, bytes_):
-        image = self.load_image_from_bytes(file_name, bytes_)
+        image = await self.load_image_from_bytes(file_name, bytes_)
         self.image = image
         await self.viewer.view_image(image, name=file_name)
         return image.shape
@@ -165,7 +163,7 @@
         self.spots = coords
         await self.viewer.add_points(
             np.flip(coords, axis=1), name="spots")
-        enh_bytes = self.save_image_to_bytes(enh_img)
+        enh_bytes = await self.save_image_to_bytes(enh_img)
         fake_csv_file = io.StringIO()
         coords_csv = df.to_csv(fake_csv_file, index=False)
         csv_bytes = fake_csv_file.getvalue().encode()

public/scripts/demo.py

@@ -1,6 +1,5 @@
-import numpy as np
-ufish = await api.createWindow("https://ufish-team.github.io/")
-img = np.random.rand(100, 100)  # generate a random image
+ufish = await api.createWindow("http://localhost:5173/")
+img = await ufish.fetchImage("https://huggingface.co/datasets/NaNg/TestData/resolve/main/FISH_spots/MERFISH_1.tif")
 out = await ufish.predict(img)  # predict the spots
 print(out.enhanced.shape)
-print(out.spots.shape)
+print(out.spots.shape)

src/main.ts

@@ -67,14 +67,24 @@ async function createApi() {
     return await getOutput()
   }
 
+  async function fetchImage(url: string) {
+    await waitRunable()
+    const runStore = useRunStore()
+    runStore.setImageUrl(url)
+    const arr = await runStore.getFetchedImage()
+    const res = removeNpArrProxy(arr)
+    return res
+  }
+
   return {
-    "run": async () => {},
+    "run": async () => {await waitRunable()},
     "setup": setup,
     "waitReady": waitRunable,
     "runPredict": run,
     "setInputImage": setInputImage,
     "getOutput": getOutput,
     "predict": predict,
+    "fetchImage": fetchImage,
   }
 }
 

src/stores/run.ts

@@ -21,6 +21,11 @@ export async function waitOutputGetable() {
   await waitState(() => store.outputGetable, true);
 }
 
+export async function waitFetchGetable() {
+  const store = useRunStore();
+  await waitState(() => store.fetchGetable, true);
+}
+
 type ApiInput = {
   data: object,
   name: string,
@@ -38,6 +43,9 @@ export const useRunStore = defineStore("run", {
     inputImage: null as ApiInput | null,
     output: null as ApiOutput | null,
     outputGetable: false,
+    imageUrl: null as string | null,
+    fetchedImage: null as object | null,
+    fetchGetable: false,
   }),
   actions: {
     async waitRunable() {
@@ -66,5 +74,19 @@ export const useRunStore = defineStore("run", {
       await waitOutputGetable();
       return this.output;
     },
+    setImageUrl(url: string | null) {
+      this.imageUrl = url;
+      if (url !== null) {
+        this.fetchGetable = false;
+      }
+    },
+    setFetchedImage(image: object | null) {
+      this.fetchedImage = image;
+      this.fetchGetable = true;
+    },
+    async getFetchedImage() {
+      await waitFetchGetable();
+      return this.fetchedImage;
+    }
   }
 }) 

src/views/PredictView.vue

@@ -211,10 +211,10 @@ export default {
       fileInput.click()
     }
 
-    async function loadExample() {
+    async function loadFromUrl(url: string) {
       loadingData.value = true
       if (plugin.value !== null) {
-        const res = await fetch(test_data_url)
+        const res = await fetch(url)
         if (!res.ok) {
           loadingData.value = false
           runInfoText.value = `Failed to load example image, status: ${res.status}`
@@ -230,15 +230,42 @@ export default {
         hasError.value = false
         output.value = null
       } else {
-        setTimeout(loadExample, 1000)
+        setTimeout(() => loadFromUrl(url), 1000)
       }
     }
 
+    const loadExample = () => loadFromUrl(test_data_url)
+
     async function download() {
       downloadBlob(output.value.enhanced, "enhanced.tif", "image/tiff")
       downloadBlob(output.value.coords, "coords.csv", "text/csv;charset=utf-8;")
     }
 
+    async function fetchImageFromUrl(url: string) {
+      const res = await fetch(url)
+      if (!res.ok) {
+        throw new Error(`Failed to fetch image from ${url}, status: ${res.status}`)
+      }
+      const data = await res.arrayBuffer()
+      const fileName = url.split('/').pop()
+      const arr = await plugin.value.load_image_from_bytes(fileName, data)
+      return arr
+    }
+
+    watch(() => runStore.imageUrl, async (newVal) => {
+      if (newVal !== null) {
+        try {
+          const arr = await fetchImageFromUrl(newVal)
+          runStore.setFetchedImage(arr)
+          runStore.setImageUrl(null)
+        } catch (error) {
+          console.log(error)
+          runStore.setFetchedImage(null)
+          runStore.setImageUrl(null)
+        }
+      }
+    })
+
     return {
       plugin,
       run,