Merge pull request #2 from TetsuakiBaba/nutjs

added mouse operation [experimental]

README.md

@@ -1,8 +1,16 @@
 # handable
-handable is an application that draws only the user's hand on the entire desktop screen. During a presentation, the user can use his or her hands to communicate various gestures.
+![](./teasers/01.gif)
+handable is an application that draws only the user's hand on the entire desktop screen. During a presentation, the user can use his or her hands to communicate various gestures. <strong>You don't need any special device as it works only with your webcam.</strong>
+
+As an experimental feature, we have implemented a mouse control function. Pinch gestures can be used for mouse click, release, and drag operations. Normally, the function is turned off, so please check the Mouse Operation checkbox in the tray menu.
+
 
-![](./teaser.gif)
 
 ## Dependencies (npm)
  * electron
- * electron-builder
+ * electron-builder
+ * @nut-tree/nut-js
+ * electron-notarize
+
+ ## Copyright
+  * author: Tetsuaki BABA

build/entitlements.mac.plist

@@ -1,18 +1,20 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
 <plist version="1.0">
-  <dict>
-    <key>com.apple.security.cs.allow-unsigned-executable-memory</key>
-    <true/>
-    <key>com.apple.security.cs.allow-jit</key>
-    <true/>
-    <key>com.apple.security.cs.allow-dyld-environment-variables</key>
-    <true/>
-    <key>com.apple.security.device.microphone</key>
-    <true/>
-    <key>com.apple.security.device.camera</key>
-    <true/>
-    <key>com.apple.security.device.audio-input</key>
-    <true/>
-  </dict>
-</plist>
+<dict>
+	<key>com.apple.security.cs.allow-unsigned-executable-memory</key>
+	<true/>
+	<key>com.apple.security.cs.allow-jit</key>
+	<true/>
+	<key>com.apple.security.cs.allow-dyld-environment-variables</key>
+	<true/>
+	<key>com.apple.security.device.microphone</key>
+	<true/>
+	<key>com.apple.security.device.camera</key>
+	<true/>
+	<key>com.apple.security.device.audio-input</key>
+	<true/>
+  <key>com.apple.security.cs.disable-library-validation</key>
+<true/>
+</dict>
+</plist>

main.js

@@ -5,9 +5,12 @@ const is_mac = process.platform === 'darwin';
 const is_linux = process.platform === 'linux';
 
 // Modules to control application life and create native browser window
-const { app, globalShortcut, BrowserWindow, screen, Tray, Menu, MenuItem, ipcMain } = require('electron')
+const { app, globalShortcut, BrowserWindow, screen, Tray, Menu, MenuItem, ipcMain, dialog } = require('electron')
 const path = require('path')
-var robot = require("robotjs");
+
+
+const { mouse, Button } = require("@nut-tree/nut-js");
+
 
 var selected_deviceId = '';
 var is_camera_active = false;
@@ -71,7 +74,7 @@ app.whenReady().then(() => {
 
   let tray_menu = Menu.buildFromTemplate([
     {
-      label: 'start/stop',
+      label: 'Start/Stop',
       accelerator: 'Ctrl+Shift+s',
       click(item, focusedWindows) {
         mainWindow.webContents.executeJavaScript(`toggleMediaPipeHands();`, true)
@@ -111,7 +114,19 @@ app.whenReady().then(() => {
       type: 'separator',
     },
 
-    { label: 'Quit handable', role: 'quit' },
+    {
+      label: 'About',
+      click(item, focusedWindows) {
+        const options = {
+          type: 'info',  // none/info/error/quetion/warning
+          title: 'ABOUT',
+          message: `${app.getName()} by Tetsuaki BABA\nVersion: ${app.getVersion()}`,
+          detail: `https://github.com/TetsuakiBaba/handable`
+        };
+        dialog.showMessageBox(options);
+      }
+    },
+    { label: 'Quit', role: 'quit' },
   ]);
 
   let screens = screen.getAllDisplays();
@@ -137,6 +152,9 @@ app.whenReady().then(() => {
         console.log(item.x, item.y, item.w, item.h);
       }
     };
+    if (sc_count == 0) {
+      data_append.submenu[sc_count].checked = true;
+    }
     sc_count++;
   }
   tray_menu.insert(2, new MenuItem(data_append));
@@ -161,14 +179,14 @@ app.whenReady().then(() => {
             if (is_camera_active) {
               mainWindow.webContents.executeJavaScript(`stopMediaPipeHands();`, true)
                 .then(result => {
-                  mainWindow.webContents.executeJavaScript(`startMediaPipeHands("${selected_deviceId}");`, true)
+                  mainWindow.webContents.executeJavaScript(`startMediaPipeHands("${selected_deviceId}"); `, true)
                     .then(result => {
                     }).catch(console.error);
                 }).catch(console.error);
             }
           }
         };
-        if (sc_count == 2) {
+        if (sc_count == 0) {
           selected_deviceId = device.id;
           data_append_camera.submenu[sc_count].checked = true;
         }
@@ -187,7 +205,7 @@ app.whenReady().then(() => {
 
   var ret = globalShortcut.register('ctrl+shift+s', function () {
     console.log('toggle MediaPipeHands');
-    mainWindow.webContents.executeJavaScript(`toggleMediaPipeHands("${selected_deviceId}");`, true)
+    mainWindow.webContents.executeJavaScript(`toggleMediaPipeHands("${selected_deviceId}"); `, true)
       .then(result => {
         is_camera_active = !is_camera_active;
         if (is_camera_active) {
@@ -235,28 +253,28 @@ app.on('will-quit', function () {
 
 
 //----------------------------------------
-// IPC通信
+// IPC Communication
 //----------------------------------------
-// 語尾に "にゃん" を付けて返す
-ipcMain.handle('nyan', (event, data) => {
-  return (`${data}にゃん`)
-})
-
 ipcMain.handle('down', (event, data) => {
   if (is_enable_mouse) {
-    robot.moveMouse(data.x, data.y);
-    robot.mouseToggle('down');
+    (async () => {
+      await mouse.setPosition(data);
+      await mouse.pressButton(Button.LEFT);
+    })()
   }
 })
 
 ipcMain.handle('drag', (event, data) => {
   if (is_enable_mouse) {
-    robot.dragMouse(data.x, data.y);
+    mouse.setPosition(data);
   }
 })
 ipcMain.handle('up', (event, data) => {
   if (is_enable_mouse) {
-    robot.moveMouse(data.x, data.y);
-    robot.mouseToggle('up');
+    (async () => {
+      await mouse.setPosition(data);
+      await mouse.releaseButton(Button.LEFT);
+
+    })()
   }
 })

package.json

@@ -1,15 +1,15 @@
 {
   "name": "handable",
-  "version": "0.2.0",
-  "description": "",
+  "version": "0.2.1",
+  "description": "handable is an application that draws only the user's hand on the entire desktop screen. During a presentation, the user can use his or her hands to communicate various gestures.",
   "main": "main.js",
   "scripts": {
     "start": "electron .",
     "build-mac": "electron-builder --mac",
     "build-win": "electron-builder --win",
     "test": "echo \"Error: no test specified\" && exit 1"
   },
-  "author": "",
+  "author": "Tetsuaki BABA",
   "license": "ISC",
   "devDependencies": {
     "electron": "^14.2.1",
@@ -18,6 +18,7 @@
   "build": {
     "appId": "net.tetsuakibaba.app.handable",
     "productName": "handable",
+    "afterSign": "./scripts/notarize.js",
     "copyright": "Copyright © 2021 Tetsuaki Baba",
     "dmg": {
       "sign": false
@@ -42,8 +43,8 @@
     }
   },
   "dependencies": {
+    "@nut-tree/nut-js": "^1.7.1-next.20211116233225",
     "dotenv": "^10.0.0",
-    "electron-notarize": "^1.1.1",
-    "robotjs": "^0.6.0"
+    "electron-notarize": "^1.1.1"
   }
 }

preload.js

@@ -2,7 +2,6 @@
 // It has the same sandbox as a Chrome extension.
 const { contextBridge, ipcRenderer } = require("electron")
 contextBridge.exposeInMainWorld('myapi', {
-    nyan: async (data) => await ipcRenderer.invoke('nyan', data),
     up: async (data) => await ipcRenderer.invoke('up', data),
     down: async (data) => await ipcRenderer.invoke('down', data),
     drag: async (data) => await ipcRenderer.invoke('drag', data)

sketch.js

@@ -67,11 +67,6 @@ function setup() {
     // video.size(1280, 720);
     // video.hide();
     noLoop();
-
-    (async () => {
-        const message = await window.myapi.nyan('はい')
-        console.log(message)  // "はいにゃん"
-    })()
 }
 
 function draw() {