support call and return res using api

README.md

@@ -1,45 +1,41 @@
 # ufish-web
 
-This template should help get you started developing with Vue 3 in Vite.
+The web interface for the U-FISH project.
 
-## Recommended IDE Setup
+Online instance: [ufish-web](https://ufish-team.github.io/#/)
 
-[VSCode](https://code.visualstudio.com/) + [Volar](https://marketplace.visualstudio.com/items?itemName=Vue.volar) (and disable Vetur) + [TypeScript Vue Plugin (Volar)](https://marketplace.visualstudio.com/items?itemName=Vue.vscode-typescript-vue-plugin).
 
-## Type Support for `.vue` Imports in TS
+## ImJoy API
 
-TypeScript cannot handle type information for `.vue` imports by default, so we replace the `tsc` CLI with `vue-tsc` for type checking. In editors, we need [TypeScript Vue Plugin (Volar)](https://marketplace.visualstudio.com/items?itemName=Vue.vscode-typescript-vue-plugin) to make the TypeScript language service aware of `.vue` types.
+Usage example:
 
-If the standalone TypeScript plugin doesn't feel fast enough to you, Volar has also implemented a [Take Over Mode](https://github.com/johnsoncodehk/volar/discussions/471#discussioncomment-1361669) that is more performant. You can enable it by the following steps:
-
-1. Disable the built-in TypeScript Extension
-    1) Run `Extensions: Show Built-in Extensions` from VSCode's command palette
-    2) Find `TypeScript and JavaScript Language Features`, right click and select `Disable (Workspace)`
-2. Reload the VSCode window by running `Developer: Reload Window` from the command palette.
-
-## Customize configuration
-
-See [Vite Configuration Reference](https://vitejs.dev/config/).
+```Python
+ufish = await api.createWindow("https://ufish-team.github.io/")
+img = np.random.rand(100, 100)  # generate a random image
+out = await ufish.predict(img)  # predict the spots
+print(out.enhanced.shape)
+print(out.spots.shape)
+```
 
-## Project Setup
+## Development
 
 ```sh
 npm install
 ```
 
-### Compile and Hot-Reload for Development
+Compile and Hot-Reload for Development
 
 ```sh
 npm run dev
 ```
 
-### Type-Check, Compile and Minify for Production
+Type-Check, Compile and Minify for Production
 
 ```sh
 npm run build
 ```
 
-### Lint with [ESLint](https://eslint.org/)
+Lint with [ESLint](https://eslint.org/)
 
 ```sh
 npm run lint

public/plugins/ufish.imjoy.html

@@ -119,7 +119,9 @@
             src="https://kaibu.org/#/app",
             window_id="kaibu-container"
         )
-        self.image = None
+        self.image = None  # input image
+        self.enhanced_image = None  # enhanced image
+        self.spots = None  # spots
 
     @staticmethod
     def load_image_from_bytes(file_name, img_bytes):
@@ -150,13 +152,17 @@
         await self.viewer.view_image(image, name=file_name)
         return image.shape
 
-    async def view_img(self, image, name="image"):
+    async def view_img(self, image, name):
+        self.image = image
         await self.viewer.view_image(image, name=name)
+        return image.shape
 
     async def process_enhanced(self, enh_img):
         await self.viewer.view_image(enh_img, name="enhanced")
         df = call_spots_local_maxima(enh_img)
         coords = df.values
+        self.enhanced_image = enh_img
+        self.spots = coords
         await self.viewer.add_points(
             np.flip(coords, axis=1), name="spots")
         enh_bytes = self.save_image_to_bytes(enh_img)

src/main.ts

@@ -11,7 +11,7 @@ import * as directives from 'vuetify/directives'
 
 import App from './App.vue'
 import router from './router'
-import { isPluginMode } from './utils'
+import { isPluginMode, removeNpArrProxy } from './utils'
 import { useRunStore } from './stores/run'
 
 window.app = {}
@@ -40,10 +40,34 @@ async function createApi() {
     await runStore.run()
   }
 
+  async function setInputImage(inputImage: object, inputName: string) {
+    const runStore = useRunStore()
+    runStore.setInputImage(inputImage, inputName)
+  }
+
+  async function getOutput() {
+    const runStore = useRunStore()
+    const out = await runStore.getOutput()
+    const res = {
+      enhanced: removeNpArrProxy(out?.enhanced),
+      spots: removeNpArrProxy(out?.spots),
+    }
+    return res
+  }
+
+  async function predict(inputImage: object) {
+    await setInputImage(inputImage, "input")
+    await run()
+    return await getOutput()
+  }
+
   return {
     "run": async () => {},
     "setup": setup,
     "runPredict": run,
+    "setInputImage": setInputImage,
+    "getOutput": getOutput,
+    "predict": predict,
   }
 }
 

src/stores/run.ts

@@ -16,19 +16,52 @@ export async function waitRunable() {
   await waitState(() => store.runable, true);
 }
 
+export async function waitOutputGetable() {
+  const store = useRunStore();
+  await waitState(() => store.outputGetable, true);
+}
+
+type ApiInput = {
+  data: object,
+  name: string,
+}
+
+type ApiOutput = {
+  enhanced: object,
+  spots: object,
+}
+
 export const useRunStore = defineStore("run", {
   state: () => ({
-    queryCount: 0,
+    runQueryCount: 0,
     runable: false,
+    inputImage: null as ApiInput | null,
+    output: null as ApiOutput | null,
+    outputGetable: false,
   }),
   actions: {
     async run() {
       await waitRunable();
-      this.queryCount += 1;
+      this.runQueryCount += 1;
       await waitRunable();
+      this.outputGetable = false;
     },
     setRunable(runable: boolean) {
       this.runable = runable;
-    }
+    },
+    setInputImage(data: object, name: string) {
+      this.inputImage = { data, name }
+    },
+    clearInputImage() {
+      this.inputImage = null;
+    },
+    setOutput(output: ApiOutput) {
+      this.output = output;
+      this.outputGetable = true;
+    },
+    async getOutput() {
+      await waitOutputGetable();
+      return this.output;
+    },
   }
 }) 

src/utils.ts

@@ -26,3 +26,8 @@ export function downloadBlob(
   pom.click();
 }
 
+export function removeNpArrProxy (parr: any) {
+  const arr = Object.assign({}, parr)
+  arr._rshape = Object.assign([], parr._rshape)
+  return arr
+}

src/views/PredictView.vue

@@ -32,7 +32,7 @@
 
 <script lang="ts">
 import * as ort from 'onnxruntime-web';
-import { getImjoyApi, isPluginMode, downloadBlob } from '@/utils';
+import { getImjoyApi, isPluginMode, downloadBlob, removeNpArrProxy } from '@/utils';
 import { ref, onMounted, watch } from 'vue';
 import { useRunStore } from '@/stores/run';
 
@@ -45,6 +45,8 @@ export default {
     const ortSession = ref(null as ort.InferenceSession | null)
     const runInfoText = ref("loading...")
     const hasError = ref(false)
+    const hasData = ref(false)
+    const output = ref(null as any)
 
     const runStore = useRunStore()
 
@@ -103,24 +105,30 @@ export default {
 
     async function run() {
       running.value = true
-      let output;
       try {
         const sImg = await plugin.value.scale_image()
         const f32data = new Float32Array(sImg._rvalue)
         const input = new ort.Tensor(
         sImg._rdtype, f32data, sImg._rshape);
-        output = await infer_2d(input)
+        const modelOut = await infer_2d(input)
         const outImg = {
           _rtype: "ndarray",
-          _rdtype: output.type,
-          _rshape: output.dims,
-          _rvalue: (output.data as Float32Array).buffer
+          _rdtype: modelOut.type,
+          _rshape: modelOut.dims,
+          _rvalue: (modelOut.data as Float32Array).buffer
         }
         const [enhBytes, coords, numSpots] = await plugin.value.process_enhanced(outImg)
-        output = {
+        output.value = {
           enhanced: enhBytes,
           coords: coords
         }
+
+        if (isPluginMode()) {
+          runStore.setOutput({
+            enhanced: outImg,
+            spots: coords,
+          })
+        }
         runInfoText.value = `Done, ${numSpots} spots detected.`
         running.value = false
       } catch (error) {
@@ -132,8 +140,8 @@ export default {
       }
     }
 
-    watch(() => runStore.queryCount, async () => {
-      if (runStore.queryCount > 0) {
+    watch(() => runStore.runQueryCount, async () => {
+      if (runStore.runQueryCount > 0) {
         console.log('running from store')
         await run()
       }
@@ -143,21 +151,44 @@ export default {
       runStore.setRunable(!running.value)
     })
 
+    watch(() => runStore.inputImage, async (newVal: any) => {
+      if (newVal !== null) {
+        try {
+          const data = removeNpArrProxy(newVal.data)
+          const shape = await plugin.value.view_img(data, newVal.name)
+          if (shape.length !== 2) {
+            runInfoText.value = `Image loaded, shape: ${shape}, but it's not 2D, please try another image.`
+            hasError.value = true
+          } else {
+            runInfoText.value = `Image loaded, shape: ${shape}`
+            hasError.value = false
+          }
+          hasData.value = true
+          output.value = null
+        } catch (error) {
+          runInfoText.value = "Failed to load image, see console for more detail."
+          console.log(error)
+          hasError.value = true
+        }
+        runStore.clearInputImage()
+      }
+    })
+
     return {
       plugin,
       run,
       running,
       hasError,
       runInfoText,
       ortSession,
+      hasData,
+      output,
     }
   },
   data: () => ({
     loadingData: false,
-    hasData: false,
     modelLoaded: false,
     test_data_url: "https://huggingface.co/datasets/NaNg/TestData/resolve/main/FISH_spots/MERFISH_1.tif",
-    output: null as any,
     showViewer: !isPluginMode(),
   }),
   computed: {
@@ -191,6 +222,7 @@ export default {
         this.loadingData = false
         this.hasData = true
         this.hasError = false
+        this.output = null
       } else {
         setTimeout(this.loadExample, 1000)
       }
@@ -224,6 +256,7 @@ export default {
           }
           this.loadingData = false
           this.hasData = true
+          this.output = null
         }
         reader.readAsArrayBuffer(file)
       }