diff --git a/pom.xml b/pom.xml index 6e8fdf4829..e6b9aa346c 100644 --- a/pom.xml +++ b/pom.xml @@ -162,7 +162,7 @@ - + org.boofcv boofcv-all @@ -198,6 +198,12 @@ + + org.nanohttpd + nanohttpd + 2.2.0 + + org.bytedeco @@ -1602,6 +1608,34 @@ + + + io.vertx + vertx-core + 4.3.3 + provided + + + io.netty + * + + + + + io.vertx + vertx-web + 4.3.3 + provided + + + io.netty + * + + + + + + @@ -1645,6 +1679,18 @@ + + au.edu.federation.caliko + caliko + 1.3.8 + + + + au.edu.federation.caliko.visualisation + caliko-visualisation + 1.3.8 + + com.github.sarxos @@ -1733,6 +1779,12 @@ + + dev.onvoid.webrtc + webrtc-java + 0.7.0 + + org.mockito @@ -1740,6 +1792,11 @@ 3.12.4 test + + au.edu.federation.caliko.demo + caliko-demo + 1.3.8 + diff --git a/src/main/java/org/myrobotlab/framework/interfaces/JsonSender.java b/src/main/java/org/myrobotlab/framework/interfaces/JsonSender.java new file mode 100644 index 0000000000..ef2a7c0056 --- /dev/null +++ b/src/main/java/org/myrobotlab/framework/interfaces/JsonSender.java @@ -0,0 +1,12 @@ +package org.myrobotlab.framework.interfaces; + +public interface JsonSender { + + /** + * Send interface which takes a json encoded Message. + * For schema look at org.myrobotlab.framework.Message + * @param jsonEncodedMessage + */ + public void send(String jsonEncodedMessage); + +} diff --git a/src/main/java/org/myrobotlab/framework/interfaces/MessageSender.java b/src/main/java/org/myrobotlab/framework/interfaces/MessageSender.java index ba0f5f1f19..c05a7b16e2 100644 --- a/src/main/java/org/myrobotlab/framework/interfaces/MessageSender.java +++ b/src/main/java/org/myrobotlab/framework/interfaces/MessageSender.java @@ -3,7 +3,7 @@ import org.myrobotlab.framework.Message; import org.myrobotlab.framework.TimeoutException; -public interface MessageSender extends NameProvider { +public interface MessageSender extends NameProvider, SimpleMessageSender { /** * Send invoking messages to remote location to invoke {name} instance's diff --git a/src/main/java/org/myrobotlab/framework/interfaces/SimpleMessageSender.java b/src/main/java/org/myrobotlab/framework/interfaces/SimpleMessageSender.java new file mode 100644 index 0000000000..a4d6d54a64 --- /dev/null +++ b/src/main/java/org/myrobotlab/framework/interfaces/SimpleMessageSender.java @@ -0,0 +1,9 @@ +package org.myrobotlab.framework.interfaces; + +import org.myrobotlab.framework.Message; + +public interface SimpleMessageSender { + + public void send(Message msg); + +} diff --git a/src/main/java/org/myrobotlab/service/FiniteStateMachine.java b/src/main/java/org/myrobotlab/service/FiniteStateMachine.java index a093f61565..d04436f1e3 100644 --- a/src/main/java/org/myrobotlab/service/FiniteStateMachine.java +++ b/src/main/java/org/myrobotlab/service/FiniteStateMachine.java @@ -17,7 +17,6 @@ import org.myrobotlab.logging.LoggingFactory; import org.myrobotlab.service.config.FiniteStateMachineConfig; import org.myrobotlab.service.config.FiniteStateMachineConfig.Transition; -import org.myrobotlab.service.config.ServiceConfig; import org.slf4j.Logger; import com.github.pnavais.machine.StateMachine; @@ -46,8 +45,14 @@ public class FiniteStateMachine extends Service { protected String lastEvent = null; + @Deprecated /* is this deprecated with ServiceConfig.listeners ? */ protected Set messageListeners = new HashSet<>(); + /** + * state history of fsm + */ + protected List history = new ArrayList<>(); + // TODO - .from("A").to("B").on(Messages.ANY) // TODO - .from("A").to("B").on(Messages.EMPTY) @@ -58,6 +63,21 @@ public class Tuple { public Transition transition; public StateTransition stateTransition; } + + public class StateChange { + public String last; + public String current; + public String event; + public StateChange(String last, String current, String event) { + this.last = last; + this.current = current; + this.event = event; + } + + public String toString() { + return String.format("%s --%s--> %s", last, event, current); + } + } private static Transition toFsmTransition(StateTransition state) { Transition transition = new Transition(); @@ -92,6 +112,13 @@ public String getNext(String key) { public void init() { stateMachine.init(); + State state = stateMachine.getCurrent(); + if (history.size() > 100) { + history.remove(0); + } + if (state != null) { + history.add(state.getName()); + } } private String makeKey(String state0, String msgType, String state1) { @@ -167,7 +194,8 @@ public void fire(String event) { log.info("fired event ({}) -> ({}) moves to ({})", event, last == null ? null : last.getName(), current == null ? null : current.getName()); if (last != null && !last.equals(current)) { - invoke("publishNewState", current.getName()); + invoke("publishStateChange", new StateChange(last.getName(), current.getName(), event)); + history.add(current.getName()); } } catch (Exception e) { log.error("fire threw", e); @@ -209,21 +237,21 @@ public List getTransitions() { } /** - * publishes state if changed here + * Publishes state change (current, last and event) * - * @param state + * @param stateChange * @return */ - public String publishNewState(String state) { - log.error("publishNewState {}", state); + public StateChange publishStateChange(StateChange stateChange) { + log.info("publishStateChange {}", stateChange); for (String listener : messageListeners) { ServiceInterface service = Runtime.getService(listener); if (service != null) { - org.myrobotlab.framework.Message msg = org.myrobotlab.framework.Message.createMessage(getName(), listener, CodecUtils.getCallbackTopicName(state), null); + org.myrobotlab.framework.Message msg = org.myrobotlab.framework.Message.createMessage(getName(), listener, CodecUtils.getCallbackTopicName(stateChange.current), null); service.in(msg); } } - return state; + return stateChange; } @Override @@ -391,7 +419,7 @@ public void setCurrent(String state) { stateMachine.setCurrent(state); current = stateMachine.getCurrent(); if (last != null && !last.equals(current)) { - invoke("publishNewState", current.getName()); + invoke("publishStateChange", new StateChange(last.getName(), current.getName(), null)); } } catch (Exception e) { log.error("setCurrent threw", e); diff --git a/src/main/java/org/myrobotlab/service/JMonkeyEngine.java b/src/main/java/org/myrobotlab/service/JMonkeyEngine.java index a22156778e..9ca917a6cf 100644 --- a/src/main/java/org/myrobotlab/service/JMonkeyEngine.java +++ b/src/main/java/org/myrobotlab/service/JMonkeyEngine.java @@ -7,8 +7,6 @@ import java.io.FileOutputStream; import java.io.IOException; import java.nio.FloatBuffer; -import java.nio.file.Path; -import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -52,7 +50,6 @@ import org.myrobotlab.net.Connection; import org.myrobotlab.sensor.EncoderData; import org.myrobotlab.sensor.EncoderListener; -import org.myrobotlab.service.abstracts.AbstractComputerVision; import org.myrobotlab.service.config.JMonkeyEngineConfig; import org.myrobotlab.service.config.ServiceConfig; import org.myrobotlab.service.interfaces.Gateway; @@ -405,10 +402,12 @@ public void attach(Attachable attachable) throws Exception { // this is to support future (non-Java) classes that cannot be instantiated // and // are subclassed in a proxy class with getType() overloaded for to identify + /**
 DO NOT NEED THIS UNTIL JMONKEY DISPLAYS VIDEO DATA - SLAM MAPPING
     if (service.getTypeKey().equals("org.myrobotlab.service.OpenCV")) {
       AbstractComputerVision cv = (AbstractComputerVision) service;
       subscribe(service.getName(), "publishCvData");
-    }
+    }
+ */ if (service.getTypeKey().equals("org.myrobotlab.service.Servo")) { // non-batched - "instantaneous" move data subscription @@ -1469,7 +1468,7 @@ public void onAnalog(String name, float keyPressed, float tpf) { // PAN -- works(ish) if (mouseMiddle && shiftLeft) { - log.info("PAN !!!!"); + log.debug("panning"); switch (name) { case "mouse-axis-x": case "mouse-axis-x-negative": diff --git a/src/main/java/org/myrobotlab/service/OakD.java b/src/main/java/org/myrobotlab/service/OakD.java index 99e9fdbfea..7c0789211c 100644 --- a/src/main/java/org/myrobotlab/service/OakD.java +++ b/src/main/java/org/myrobotlab/service/OakD.java @@ -1,10 +1,12 @@ package org.myrobotlab.service; import org.myrobotlab.framework.Service; +import org.myrobotlab.framework.Status; import org.myrobotlab.logging.Level; import org.myrobotlab.logging.LoggerFactory; import org.myrobotlab.logging.LoggingFactory; -import org.myrobotlab.service.config.ServiceConfig; +import org.myrobotlab.process.GitHub; +import org.myrobotlab.service.config.OakDConfig; import org.slf4j.Logger; /** * @@ -14,16 +16,50 @@ * @author GroG * */ -public class OakD extends Service { +public class OakD extends Service { private static final long serialVersionUID = 1L; public final static Logger log = LoggerFactory.getLogger(OakD.class); + private transient Py4j py4j = null; + private transient Git git = null; + public OakD(String n, String id) { super(n, id); } + public void startService() { + super.startService(); + + py4j = (Py4j)startPeer("py4j"); + git = (Git)startPeer("git"); + + if (config.py4jInstall) { + installDepthAi(); + } + + } + + /** + * starting install of depthapi + */ + public void publishInstallStart() { + } + + public Status publishInstallFinish() { + return Status.error("depth ai install was not successful"); + } + + /** + * For depthai we need to clone its repo and install requirements + * + */ + public void installDepthAi() { + + //git.clone("./", config.depthaiCloneUrl) + py4j.exec(""); + } public static void main(String[] args) { try { diff --git a/src/main/java/org/myrobotlab/service/ProgramAB.java b/src/main/java/org/myrobotlab/service/ProgramAB.java index 29b8653ad8..db06861aaa 100644 --- a/src/main/java/org/myrobotlab/service/ProgramAB.java +++ b/src/main/java/org/myrobotlab/service/ProgramAB.java @@ -792,7 +792,7 @@ public String addBotPath(String path) { broadcastState(); } else { - error("invalid bot path - a bot must be a directory with a subdirectory named \"aiml\""); + error("invalid bot path %s - a bot must be a directory with a subdirectory named \"aiml\"", path); return null; } return path; diff --git a/src/main/java/org/myrobotlab/service/Py4j.java b/src/main/java/org/myrobotlab/service/Py4j.java index 8c5e59b502..909270e77c 100644 --- a/src/main/java/org/myrobotlab/service/Py4j.java +++ b/src/main/java/org/myrobotlab/service/Py4j.java @@ -9,7 +9,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import org.bytedeco.javacpp.Loader; import org.myrobotlab.codec.CodecUtils; @@ -21,9 +20,11 @@ import org.myrobotlab.logging.Level; import org.myrobotlab.logging.LoggerFactory; import org.myrobotlab.logging.LoggingFactory; +import org.myrobotlab.net.Connection; import org.myrobotlab.service.config.Py4jConfig; import org.myrobotlab.service.data.Script; import org.myrobotlab.service.interfaces.Executor; +import org.myrobotlab.service.interfaces.Gateway; import org.slf4j.Logger; import py4j.GatewayServer; @@ -53,7 +54,7 @@ * * @author GroG */ -public class Py4j extends Service implements GatewayServerListener { +public class Py4j extends Service implements GatewayServerListener, Gateway { /** * POJO class to tie all the data elements of a external python process @@ -234,15 +235,6 @@ private String getClientKey(Py4JServerConnection gatewayConnection) { return String.format("%s:%d", gatewayConnection.getSocket().getInetAddress(), gatewayConnection.getSocket().getPort()); } - /** - * return a set of client connections - probably could be deprecated to a - * single client, but was not sure - * - * @return - */ - public Set getClients() { - return clients.keySet(); - } /** * get listing of filesystem files location will be data/Py4j/{serviceName} @@ -336,7 +328,19 @@ public boolean preProcessHook(Message msg) { // TODO - determine clients are connected .. how many clients etc.. try { if (handler != null) { - handler.invoke(msg.method, msg.data); + // afaik - Py4j does some kind of magical encoding to get a JavaObject + // back to the Python process, but: + // 1. its useless for users - no way to access the content ? + // 2. you can't do anything with it + // So, I've chosen to json encode it here, and the Py4j.py MessageHandler will + // decode it into a Python dictionary \o/ + // we do single encoding including the parameter array - there is no header needed + // with method and other details, as the invoke here is invoking directly in the + // Py4j.py script + + String json = CodecUtils.toJson(msg); + // handler.invoke(msg.method, json); + handler.send(json); } else { error("preProcessHook handler is null"); } @@ -588,7 +592,37 @@ public static void main(String[] args) { log.error("main threw", e); } } - - + + @Override + public void connect(String uri) throws Exception { + // host:port of python process running py4j ??? + + } + + /** + * Remote in this context is the remote python process + */ + @Override + public void sendRemote(Message msg) throws Exception { + log.info("sendRemote"); + String jsonMsg = CodecUtils.toJson(msg); + handler.send(jsonMsg); + } + + @Override + public boolean isLocal(Message msg) { + return Runtime.getInstance().isLocal(msg); + } + + @Override + public List getClientIds() { + return Runtime.getInstance().getConnectionUuids(getName()); + } + + @Override + public Map getClients() { + return Runtime.getInstance().getConnections(getName()); + } + } diff --git a/src/main/java/org/myrobotlab/service/Vertx.java b/src/main/java/org/myrobotlab/service/Vertx.java new file mode 100644 index 0000000000..09cd1132ad --- /dev/null +++ b/src/main/java/org/myrobotlab/service/Vertx.java @@ -0,0 +1,174 @@ +package org.myrobotlab.service; + +import java.net.URISyntaxException; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.myrobotlab.codec.CodecUtils; +import org.myrobotlab.framework.Message; +import org.myrobotlab.framework.Service; +import org.myrobotlab.logging.Level; +import org.myrobotlab.logging.LoggerFactory; +import org.myrobotlab.logging.LoggingFactory; +import org.myrobotlab.net.Connection; +import org.myrobotlab.service.config.VertxConfig; +import org.myrobotlab.service.interfaces.Gateway; +import org.myrobotlab.vertx.ApiVerticle; +import org.slf4j.Logger; + +import io.vertx.core.VertxOptions; +import io.vertx.core.http.ServerWebSocket; + +/** + * Vertx gateway - used to support a http and websocket gateway for myrobotlab. + * Write business logic in Verticles. Also, try not to write any logic besides initialization inside start() method. + * + * It currently does not utilize the Vertx event bus - which is pretty much the most important part of Vertx. + * TODO: take advantage of publishing on the event bus + * + * @see https://medium.com/@pvub/https-medium-com-pvub-vert-x-workers-6a8df9b2b9ee + * + * @author GroG + * + */ +public class Vertx extends Service implements Gateway { + + private static final long serialVersionUID = 1L; + + private transient io.vertx.core.Vertx vertx = null; + + public final static Logger log = LoggerFactory.getLogger(Vertx.class); + + public Vertx(String n, String id) { + super(n, id); + } + + /** + * deploys a http and websocket verticle on a secure TLS channel with self signed certificate + */ + public void start() { + log.info("starting driver"); + + /** + * FIXME - might have to revisit this This is a block comment, but takes + * advantage of javadoc pre non-formatting in ide to preserve the code + * formatting + * + * 
+     * 
+     * final Vertx that = this;
+     * 
+     * java.lang.Runtime.getRuntime().addShutdownHook(new Thread() {
+     *   public void run() {
+     *     System.out.println("Running Shutdown Hook");
+     *     that.stop();
+     *   }
+     * });
+     * 
+     *
+ */ + + vertx = io.vertx.core.Vertx.vertx(new VertxOptions().setBlockedThreadCheckInterval(100000)); + vertx.deployVerticle(new ApiVerticle(this)); + + } + + @Override + public void startService() { + super.startService(); + start(); + } + + @Override + public void stopService() { + super.stopService(); + stop(); + } + + /** + * Undeploy the verticle serving http and ws + */ + public void stop() { + log.info("stopping driver"); + Set ids = vertx.deploymentIDs(); + for (String id : ids) { + vertx.undeploy(id, (result) -> { + if (result.succeeded()) { + log.info("undeploy succeeded"); + } else { + log.error("undeploy failed"); + } + }); + } + } + + public static void main(String[] args) { + try { + + LoggingFactory.init(Level.INFO); + + Vertx vertx = (Vertx) Runtime.start("vertx", "Vertx"); + vertx.start(); + + InMoov2 i01 = (InMoov2)Runtime.start("i01", "InMoov2"); + // i01.startSimulator(); + JMonkeyEngine jme = (JMonkeyEngine)i01.startPeer("simulator"); +// Runtime.start("python", "Python"); +// + WebGui webgui = (WebGui) Runtime.create("webgui", "WebGui"); + // webgui.setSsl(true); + webgui.autoStartBrowser(false); + webgui.setPort(8888); + webgui.startService(); + + } catch (Exception e) { + log.error("main threw", e); + } + } + + // FIXME - refactor for bare minimum + + @Override /* FIXME "Gateway" is server/service oriented not connecting thing - remove this */ + public void connect(String uri) throws URISyntaxException { + // TODO Auto-generated method stub + + } + + @Override /* FIXME not much point of these - as they are all consistently using Runtime's centralized connection info */ + public List getClientIds() { + return Runtime.getInstance().getConnectionUuids(getName()); + } + + @Override /* FIXME not much point of these - as they are all consistently using Runtime's centralized connection info */ + public Map getClients() { + return Runtime.getInstance().getConnections(getName()); + } + + + @Override /* FIXME this is the one and probably "only" relevant method for Gateway - perhaps a handle(Connection c) */ + public void sendRemote(Message msg) throws Exception { + log.info("sendRemote {}", msg.toString()); + // FIXME MUST BE DIRECT THREAD FROM BROADCAST NOT OUTBOX !!! + msg.addHop(getId()); + Map clients = getClients(); + for(Connection c: clients.values()) { + try { + ServerWebSocket socket = (ServerWebSocket)c.get("websocket"); + String json = CodecUtils.toJsonMsg(msg); + socket.writeTextMessage(json); + } catch(Exception e) { + error(e); + } + } + // broadcastMode - iterate through clients send all + } + + @Override + public boolean isLocal(Message msg) { + return Runtime.getInstance().isLocal(msg); } + + public io.vertx.core.Vertx getVertx() { + return vertx; + } +} diff --git a/src/main/java/org/myrobotlab/service/WebXR.java b/src/main/java/org/myrobotlab/service/WebXR.java new file mode 100644 index 0000000000..509620832e --- /dev/null +++ b/src/main/java/org/myrobotlab/service/WebXR.java @@ -0,0 +1,163 @@ +package org.myrobotlab.service; + +import java.util.HashMap; +import java.util.Map; + +import org.myrobotlab.framework.Service; +import org.myrobotlab.kinematics.Point; +import org.myrobotlab.logging.Level; +import org.myrobotlab.logging.LoggerFactory; +import org.myrobotlab.logging.LoggingFactory; +import org.myrobotlab.math.MapperSimple; +import org.myrobotlab.service.config.WebXRConfig; +import org.myrobotlab.service.data.Event; +import org.myrobotlab.service.data.Pose; +import org.slf4j.Logger; + +public class WebXR extends Service { + + private static final long serialVersionUID = 1L; + + public final static Logger log = LoggerFactory.getLogger(WebXR.class); + + public WebXR(String n, String id) { + super(n, id); + } + + public Event publishEvent(Event event) { + if (log.isDebugEnabled()) { + log.debug("publishEvent {}", event); + } + + String path = String.format("event.%s.%s", event.meta.get("handedness"), event.type); + if (event.value != null) { + path = path + "." + event.value.toString(); + } + + if (config.eventMappings.containsKey(path)) { + // TODO - future might be events -> message that goes to ServoMixer .e.g mixer.playGesture("closeHand") + // or sadly Python execute script for total chaos :P + invoke("publishJointAngles", config.eventMappings.get(path)); + } + + return event; + } + + /** + * Pose is the x,y,z and pitch, roll, yaw of all the devices WebXR found. + * Hopefully, this includes headset, and hand controllers. + * WebXRConfig processes a mapping between these values (usually in radians) to + * servo positions, and will then publish JointAngles for servos. + * + * @param pose + * @return + */ + public Pose publishPose(Pose pose) { + if (log.isDebugEnabled()) { + log.debug("publishPose {}", pose); + } + // process mappings config into joint angles + Map map = new HashMap<>(); + + String path = String.format("%s.orientation.roll", pose.name); + if (config.controllerMappings.containsKey(path)) { + Map mapper = config.controllerMappings.get(path); + for (String name : mapper.keySet()) { + map.put(name, mapper.get(name).calcOutput(pose.orientation.roll)); + } + } + + path = String.format("%s.orientation.pitch", pose.name); + if (config.controllerMappings.containsKey(path)) { + Map mapper = config.controllerMappings.get(path); + for (String name : mapper.keySet()) { + map.put(name, mapper.get(name).calcOutput(pose.orientation.pitch)); + } + } + + path = String.format("%s.orientation.yaw", pose.name); + if (config.controllerMappings.containsKey(path)) { + Map mapper = config.controllerMappings.get(path); + for (String name : mapper.keySet()) { + map.put(name, mapper.get(name).calcOutput(pose.orientation.yaw)); + } + } + + InverseKinematics3D ik = (InverseKinematics3D)Runtime.getService("ik3d"); + if (ik != null && pose.name.equals("left")) { + ik.setCurrentArm("left", InMoov2Arm.getDHRobotArm("i01", "left")); + + ik.centerAllJoints("left"); + + for (int i = 0; i < 1000; ++i) { + + ik.centerAllJoints("left"); + ik.moveTo("left", 0, 0.0+ i * 0.02, 0.0); + + + // ik.moveTo(pose.name, new Point(0, -200, 50)); + } + + // map name + // and then map all position and rotation too :P + Point p = new Point(70 + pose.position.x, -550 + pose.position.y, pose.position.z); + + ik.moveTo(pose.name, p); + } + + if (map.size() > 0) { + invoke("publishJointAngles", map); + } + + // TODO - publishQuaternion + // invoke("publishQuaternion", map); + + return pose; + } + + public Map publishJointAngles(Map map) { + for (String name: map.keySet()) { + log.info("{}.moveTo {}", name, map.get(name)); + } + return map; + } + + public static void main(String[] args) { + try { + + LoggingFactory.init(Level.INFO); + + // identical to command line start + // Runtime.startConfig("inmoov2"); + + + // normal non-config launch + // Runtime.main(new String[] { "--log-level", "info", "-s", "webgui", "WebGui", "intro", "Intro", "python", "Python" }); + + + // config launch + Runtime.startConfig("webxr"); + + boolean done = true; + if (done) + return; + + Runtime.startConfig("webxr"); + boolean done2 = true; + if (done2) + return; + + Runtime.start("webxr", "WebXR"); + WebGui webgui = (WebGui) Runtime.create("webgui", "WebGui"); + // webgui.setSsl(true); + webgui.autoStartBrowser(false); + webgui.startService(); + Runtime.start("vertx", "Vertx"); + InMoov2 i01 = (InMoov2) Runtime.start("i01", "InMoov2"); + i01.startPeer("simulator"); + + } catch (Exception e) { + log.error("main threw", e); + } + } +} diff --git a/src/main/java/org/myrobotlab/service/config/OakDConfig.java b/src/main/java/org/myrobotlab/service/config/OakDConfig.java new file mode 100644 index 0000000000..6a7295da1c --- /dev/null +++ b/src/main/java/org/myrobotlab/service/config/OakDConfig.java @@ -0,0 +1,32 @@ +package org.myrobotlab.service.config; + +import org.myrobotlab.framework.Plan; + +public class OakDConfig extends ServiceConfig { + + + /** + * install through py4j + */ + public boolean py4jInstall = true; + + /** + * the depthai clone + */ + public String depthaiCloneUrl = "https://github.com/luxonis/depthai.git"; + + /** + * pin the repo + */ + public String depthAiSha = "dde0ba57dba673f67a62e4fb080f22d6cfcd3224"; + + @Override + public Plan getDefault(Plan plan, String name) { + super.getDefault(plan, name); + addDefaultPeerConfig(plan, name, "py4j", "Py4j"); + addDefaultPeerConfig(plan, name, "git", "Git"); + return plan; + } + + +} diff --git a/src/main/java/org/myrobotlab/service/config/VertxConfig.java b/src/main/java/org/myrobotlab/service/config/VertxConfig.java new file mode 100644 index 0000000000..f2119d8ddd --- /dev/null +++ b/src/main/java/org/myrobotlab/service/config/VertxConfig.java @@ -0,0 +1,9 @@ +package org.myrobotlab.service.config; + +public class VertxConfig extends ServiceConfig { + + public Integer port = 8443; + public Integer workerCount = 1; + public boolean ssl = true; + +} diff --git a/src/main/java/org/myrobotlab/service/config/WebXRConfig.java b/src/main/java/org/myrobotlab/service/config/WebXRConfig.java new file mode 100644 index 0000000000..2990f1c60d --- /dev/null +++ b/src/main/java/org/myrobotlab/service/config/WebXRConfig.java @@ -0,0 +1,37 @@ +package org.myrobotlab.service.config; + +import java.util.HashMap; +import java.util.Map; + +import org.myrobotlab.framework.Plan; +import org.myrobotlab.math.MapperSimple; + +public class WebXRConfig extends ServiceConfig { + + public Map> eventMappings = new HashMap<>(); + + public Map> controllerMappings = new HashMap<>(); + + + public Plan getDefault(Plan plan, String name) { + super.getDefault(plan, name); + + Map map = new HashMap<>(); + MapperSimple mapper = new MapperSimple(-0.5, 0.5, 0, 180); + map.put("i01.head.neck", mapper); + controllerMappings.put("head.orientation.pitch", map); + + map = new HashMap<>(); + mapper = new MapperSimple(-0.5, 0.5, 0, 180); + map.put("i01.head.rothead", mapper); + controllerMappings.put("head.orientation.yaw", map); + + map = new HashMap<>(); + mapper = new MapperSimple(-0.5, 0.5, 0, 180); + map.put("i01.head.roll", mapper); + controllerMappings.put("head.orientation.roll", map); + + return plan; + } + +} diff --git a/src/main/java/org/myrobotlab/service/data/Event.java b/src/main/java/org/myrobotlab/service/data/Event.java new file mode 100644 index 0000000000..9a3572035c --- /dev/null +++ b/src/main/java/org/myrobotlab/service/data/Event.java @@ -0,0 +1,38 @@ +package org.myrobotlab.service.data; + +import java.util.Map; + +/** + * Generalized Event POJO class, started by WebXR but intended to be used + * in other event generating services + * + * @author GroG + * + */ +public class Event { + /** + * Identifier of the event typically its the identifier of some + * detail of the source of the event e.g. in WebXR it is the uuid + */ + public String id; + + /** + * type of event WebXR has sqeezestart sqeezeend and others + */ + public String type; + + + /** + * Value of the event, could be string or numeric or boolean + */ + public Object value; + + + /** + * Meta data regarding the event, could be additional values like + * "handedness" in WebXR + */ + public Map meta; + + +} diff --git a/src/main/java/org/myrobotlab/service/data/Orientation.java b/src/main/java/org/myrobotlab/service/data/Orientation.java index b2d5d5658e..b7df4102dc 100644 --- a/src/main/java/org/myrobotlab/service/data/Orientation.java +++ b/src/main/java/org/myrobotlab/service/data/Orientation.java @@ -11,6 +11,7 @@ public class Orientation { public Double roll = null; public Double pitch = null; public Double yaw = null; + public String src = null; // default constructor (values will be null until set) public Orientation() { diff --git a/src/main/java/org/myrobotlab/service/data/Pose.java b/src/main/java/org/myrobotlab/service/data/Pose.java new file mode 100644 index 0000000000..767d9be81d --- /dev/null +++ b/src/main/java/org/myrobotlab/service/data/Pose.java @@ -0,0 +1,22 @@ +package org.myrobotlab.service.data; + +public class Pose { + public String name = null; + public Long ts = null; + public Position position = null; + public Orientation orientation = null; + + public String toString() { + StringBuilder sb = new StringBuilder(); + sb.append(String.format("name:%s", name)); + if (position != null) { + sb.append(String.format(" x:%.2f y:%.2f z:%.2f", position.x, position.y, position.z)); + } + if (orientation != null) { + sb.append(String.format(" roll:%.2f pitch:%.2f yaw:%.2f", orientation.roll, orientation.pitch, orientation.yaw)); + } + return sb.toString(); + } + + +} diff --git a/src/main/java/org/myrobotlab/service/data/Position.java b/src/main/java/org/myrobotlab/service/data/Position.java new file mode 100644 index 0000000000..83fe574a44 --- /dev/null +++ b/src/main/java/org/myrobotlab/service/data/Position.java @@ -0,0 +1,43 @@ +package org.myrobotlab.service.data; + +public class Position { + + public Double x; + public Double y; + public Double z; + public String src; + + public Position(double x, double y, double z) { + this.x = x; + this.y = y; + this.z = z; + } + + public Position(double x, double y) { + this.x = x; + this.y = y; + } + + public Position(int x, int y, int z) { + this.x = (double) x; + this.y = (double) y; + this.z = (double) z; + } + + public Position(int x, int y) { + this.x = (double) x; + this.y = (double) y; + } + + public Position(float x, float y, float z) { + this.x = (double) x; + this.y = (double) y; + this.z = (double) z; + } + + public Position(float x, float y) { + this.x = (double) x; + this.y = (double) y; + } + +} diff --git a/src/main/java/org/myrobotlab/service/interfaces/Executor.java b/src/main/java/org/myrobotlab/service/interfaces/Executor.java index e1566c015a..fc45803c10 100644 --- a/src/main/java/org/myrobotlab/service/interfaces/Executor.java +++ b/src/main/java/org/myrobotlab/service/interfaces/Executor.java @@ -1,6 +1,6 @@ package org.myrobotlab.service.interfaces; -import org.myrobotlab.framework.interfaces.Invoker; +import org.myrobotlab.framework.interfaces.JsonSender; /** * Interface to a Executor - currently only utilized by Py4j to @@ -10,7 +10,7 @@ * @author GroG * */ -public interface Executor extends Invoker { +public interface Executor extends JsonSender { /** * exec in Python - executes arbitrary code diff --git a/src/main/java/org/myrobotlab/service/meta/WebXRMeta.java b/src/main/java/org/myrobotlab/service/meta/WebXRMeta.java new file mode 100644 index 0000000000..36efe701ad --- /dev/null +++ b/src/main/java/org/myrobotlab/service/meta/WebXRMeta.java @@ -0,0 +1,33 @@ +package org.myrobotlab.service.meta; + +import org.myrobotlab.logging.LoggerFactory; +import org.myrobotlab.service.meta.abstracts.MetaData; +import org.slf4j.Logger; + +public class WebXRMeta extends MetaData { + private static final long serialVersionUID = 1L; + public final static Logger log = LoggerFactory.getLogger(WebXRMeta.class); + + /** + * This class is contains all the meta data details of a service. It's peers, + * dependencies, and all other meta data related to the service. + * + */ + public WebXRMeta() { + + // add a cool description + addDescription("WebXR allows hmi devices to add input and get data back from mrl"); + + // false will prevent it being seen in the ui + setAvailable(true); + + // add it to one or many categories + addCategory("remote","control"); + + // add a sponsor to this service + // the person who will do maintenance + // setSponsor("GroG"); + + } + +} diff --git a/src/main/java/org/myrobotlab/vertx/ApiVerticle.java b/src/main/java/org/myrobotlab/vertx/ApiVerticle.java new file mode 100644 index 0000000000..4dc470f1df --- /dev/null +++ b/src/main/java/org/myrobotlab/vertx/ApiVerticle.java @@ -0,0 +1,104 @@ +package org.myrobotlab.vertx; + +import org.myrobotlab.service.config.VertxConfig; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import io.vertx.core.AbstractVerticle; +import io.vertx.core.http.HttpMethod; +import io.vertx.core.http.HttpServer; +import io.vertx.core.http.HttpServerOptions; +import io.vertx.core.net.SelfSignedCertificate; +import io.vertx.ext.web.Router; +import io.vertx.ext.web.handler.CorsHandler; +import io.vertx.ext.web.handler.StaticHandler; + +/** + * verticle to handle api requests + * + * @author GroG + */ +public class ApiVerticle extends AbstractVerticle { + + public final static Logger log = LoggerFactory.getLogger(ApiVerticle.class); + + private Router router; + + transient private org.myrobotlab.service.Vertx service; + + public ApiVerticle(org.myrobotlab.service.Vertx service) { + super(); + this.service = service; + } + + @Override + public void start() throws Exception { + // process configuration and create handlers + log.info("starting api verticle"); + VertxConfig config = (VertxConfig) service.getConfig(); + + // create a router + router = Router.router(vertx); + + // handle cors requests + router.route().handler(CorsHandler.create("*").allowedMethod(HttpMethod.GET).allowedMethod(HttpMethod.OPTIONS).allowedHeader("Accept").allowedHeader("Authorization") + .allowedHeader("Content-Type")); + + // static file routing - this is from a npm "build" ... but durin gdevelop its far + // easier to use setupProxy.js from a npm start .. but deployment would be easier with a "build" + + // new UI + // StaticHandler root = StaticHandler.create("../robotlab-x-app/build/"); + + // old UI (runtime vs dev time...) + StaticHandler root = StaticHandler.create("src/main/resources/resource/WebGui/app/"); + root.setCachingEnabled(false); + root.setDirectoryListing(true); + root.setIndexPage("index.html"); + // root.setAllowRootFileSystemAccess(true); + // root.setWebRoot(null); + + // VideoStreamHandler video = new VideoStreamHandler(service); + + // router.route("/video/*").handler(video); + router.route("/*").handler(root); + + + // router.get("/health").handler(this::generateHealth); + // router.get("/api/transaction/:customer/:tid").handler(this::handleTransaction); + + // create the HTTP server and pass the + // "accept" method to the request handler + HttpServerOptions httpOptions = new HttpServerOptions(); + + if (config.ssl) { + SelfSignedCertificate certificate = SelfSignedCertificate.create(); + httpOptions.setSsl(true); + httpOptions.setKeyCertOptions(certificate.keyCertOptions()); + httpOptions.setTrustOptions(certificate.trustOptions()); + } + httpOptions.setPort(config.port); + + + HttpServer server = vertx.createHttpServer(httpOptions); + // TODO - this is where multiple workers would be defined + // .createHttpServer() + + // WebSocketHandler webSocketHandler = new WebSocketHandler(service); + // server.webSocketHandler(webSocketHandler); + + // FIXME - don't do "long" or "common" processing in the start() + // FIXME - how to do this -> server.webSocketHandler(this::handleWebSocket); + server.webSocketHandler(new WebSocketHandler(service)); + server.requestHandler(router); + // start servers + server.listen(); + } + + + @Override + public void stop() throws Exception { + log.info("stopping api verticle"); + } + +} diff --git a/src/main/java/org/myrobotlab/vertx/WebSocketHandler.java b/src/main/java/org/myrobotlab/vertx/WebSocketHandler.java new file mode 100644 index 0000000000..69b56957f3 --- /dev/null +++ b/src/main/java/org/myrobotlab/vertx/WebSocketHandler.java @@ -0,0 +1,126 @@ +package org.myrobotlab.vertx; + +import java.lang.reflect.Method; + +import org.myrobotlab.codec.CodecUtils; +import org.myrobotlab.framework.MethodCache; +import org.myrobotlab.framework.interfaces.ServiceInterface; +import org.myrobotlab.logging.LoggerFactory; +import org.myrobotlab.net.Connection; +import org.myrobotlab.service.Runtime; +import org.slf4j.Logger; + +import io.vertx.core.Handler; +import io.vertx.core.http.ServerWebSocket; + +/** + * Minimal Handler for all websocket messages coming from the react js client. + * + * TODO - what else besides text messages - websocket binary streams ??? text + * stream ? + * + * @author GroG + * + */ +public class WebSocketHandler implements Handler { + + public final static Logger log = LoggerFactory.getLogger(WebSocketHandler.class); + + /** + * reference to the MRL Vertx service / websocket and http server + */ + transient private org.myrobotlab.service.Vertx service = null; + + /** + * reference to the websocket text message handler + */ + TextMessageHandler textMessageHandler = null; + + public static class TextMessageHandler implements Handler { + + org.myrobotlab.service.Vertx service = null; + + public TextMessageHandler(org.myrobotlab.service.Vertx service) { + this.service = service; + } + + @Override + public void handle(String json) { + log.info("handling {}", json); + + Method method; + try { + + org.myrobotlab.framework.Message msg = CodecUtils.fromJson(json, org.myrobotlab.framework.Message.class); + + Class clazz = Runtime.getClass(msg.name); + if (clazz == null) { + log.error("cannot derive local type from service {}", msg.name); + return; + } + + MethodCache cache = MethodCache.getInstance(); + Object[] params = cache.getDecodedJsonParameters(clazz, msg.method, msg.data); + + method = cache.getMethod(clazz, msg.method, params); + if (method == null) { + service.error("method cache could not find %s.%s(%s)", clazz.getSimpleName(), msg.method, msg.data); + return; + } + + // FIXME - probably shouldn't be invoking, probable should be putting + // the message on the out queue ... not sure + ServiceInterface si = Runtime.getService(msg.name); + // Object ret = method.invoke(si, params); + + // put msg on mrl msg bus :) + // service.in(msg); <- NOT DECODE PARAMS !! + + // if ((new Random()).nextInt(100) == 0) { + // ctx.close(); - will close the websocket !!! + // } else { + // ctx.writeTextMessage("ping"); Useful is writing back + // } + + // replace with typed parameters + msg.data = params; + // queue the message + si.in(msg); + + } catch (Exception e) { + service.error(e); + } + } + } + + public WebSocketHandler(org.myrobotlab.service.Vertx service) { + this.service = service; + this.textMessageHandler = new TextMessageHandler(service); + } + + @Override + public void handle(ServerWebSocket socket) { + // FIXME - get "id" from js client - need something unique from the js + // client + // String id = r.getRequest().getParameter("id"); + String id = String.format("vertx-%s", service.getName()); + // String uuid = UUID.randomUUID().toString(); + String uuid = socket.binaryHandlerID(); + Connection connection = new Connection(uuid, id, service.getName()); + connection.put("c-type", service.getSimpleName()); + connection.put("gateway", service.getName()); + connection.putTransient("websocket", socket); + Runtime.getInstance().addConnection(uuid, id, connection); + // ctx.writeTextMessage("ping"); FIXME - query ? + // FIXME - thread-safe ? how many connections mapped to objects ? + socket.textMessageHandler(textMessageHandler); + log.info("new ws connection {}", uuid); + + socket.closeHandler(close -> { + log.info("closing {}", socket.binaryHandlerID()); + Runtime.getInstance().removeConnection(socket.binaryHandlerID()); + }); + + } + +} diff --git a/src/main/resources/resource/Py4j/Py4j.py b/src/main/resources/resource/Py4j/Py4j.py index 0cdb5b3bce..3c75cd94c5 100644 --- a/src/main/resources/resource/Py4j/Py4j.py +++ b/src/main/resources/resource/Py4j/Py4j.py @@ -13,13 +13,78 @@ # the gateway import sys - +import json +from abc import ABC, abstractmethod from py4j.java_collections import JavaObject, JavaClass from py4j.java_gateway import JavaGateway, CallbackServerParameters, GatewayParameters -runtime = None +class Service(ABC): + def __init__(self, name): + self.java_object = runtime.start(name, self.getType()) + + def __getattr__(self, attr): + # Delegate attribute access to the underlying Java object + return getattr(self.java_object, attr) + + def __str__(self): + # Delegate string representation to the underlying Java object + return str(self.java_object) + + def subscribe(self, event): + print("subscribe") + self.java_object.subscribe(event) + + @abstractmethod + def getType(self): + pass + + +class NeoPixel(Service): + def __init__(self, name): + super().__init__(name) + + def getType(self): + return "NeoPixel" + + def onFlash(self): + print("onFlash") + + +class InMoov2(Service): + def __init__(self, name): + super().__init__(name) + self.subscribe('onStateChange') + + def getType(self): + return "InMoov2" + + def onOnStateChange(self, state): + print("onOnStateChange") + print(state) + print(state.get('last')) + print(state.get('current')) + print(state.get('event')) + + +# TODO dynamically add classes that you don't bother to check in +# class Runtime(Service): +# def __init__(self, name): +# super().__init__(name) + + +# FIXME - REMOVE THIS - DO NOT SET ANY GLOBALS !!!! +runtime = None + +# TODO - rename to mrl_lib ? +# e.g. +# mrl = mrl_lib.connect("localhost", 1099) +# i01 = InMoov("i01", mrl) +# or +# runtime = mrl_lib.connect("localhost", 1099) # JVM connection Py4j instance needed for a gateway +# runtime.start("i01", "InMoov2") # starts Java service +# runtime.start("nativePythonService", "NativePythonClass") # starts Python service no gateway needed class MessageHandler(object): """ The class responsible for receiving and processing Py4j messages, @@ -41,10 +106,34 @@ def __init__(self): python_server_entry_point=self, gateway_parameters=GatewayParameters(auto_convert=True)) self.runtime = self.gateway.jvm.org.myrobotlab.service.Runtime.getInstance() + # FIXME - REMOVE THIS - DO NOT SET ANY GLOBALS !!!! runtime = self.runtime self.py4j = None # need to wait until name is set print("initialized ... waiting for name to be set") + def construct_runtime(self): + """ + Constructs a new Runtime instance and returns it. + """ + jvm_runtime = self.gateway.jvm.org.myrobotlab.service.Runtime.getInstance() + + # Define class attributes and methods as dictionaries + class_attributes = { + 'x': 0, + 'y': 0, + 'move': lambda self, dx, dy: setattr(self, 'x', self.x + dx) or setattr(self, 'y', self.y + dy), + 'get_position': lambda self: (self.x, self.y), + } + + # Create the class dynamically using the type() function + MyDynamicClass = type('MyDynamicClass', (object,), class_attributes) + + # Create an instance of the dynamically created class + obj = MyDynamicClass() + + + return self.runtime + # Define the callback function def handle_connection_break(self): # Add your custom logic here to handle the connection break @@ -78,11 +167,15 @@ def setName(self, name): print("reference to runtime") # TODO print env vars PYTHONPATH etc return name + + def getRuntime(self): + return self.runtime def exec(self, code): """ Executes Python code in the global namespace. - All exceptions are caught and printed so that the Python subprocess doesn't crash. + All exceptions are caught and printed so that the + Python subprocess doesn't crash. :param code: The Python code to execute. :type code: str @@ -94,22 +187,38 @@ def exec(self, code): except Exception as e: print(e) - def invoke(self, method, data=()): + def send(self, json_msg): + msg = json.loads(json_msg) + if msg.get("data") is None or msg.get("data") == []: + globals()[msg.get("method")]() + else: + globals()[msg.get("method")](*msg.get("data")) + + # equivalent to JS onMessage + def invoke(self, method, data=None): """ Invoke a function from the global namespace with the given parameters. :param method: The name of the function to invoke. :type method: str - :param data: The parameters to pass to the function, defaulting to no parameters. + :param data: The parameters to pass to the function, defaulting to + no parameters. :type data: Iterable """ # convert to list - params = list(data) + # params = list(data) not necessary will always be a json string # Lookup the method in the global namespace # Much much faster than using eval() - globals()[method](*params) + + # data should be None or always a list of params + if data is None: + globals()[method]() + else: + # one shot json decode + params = json.loads(data) + globals()[method](*params) def shutdown(self): """ diff --git a/src/main/resources/resource/WebGui/app/service/js/FiniteStateMachineGui.js b/src/main/resources/resource/WebGui/app/service/js/FiniteStateMachineGui.js index 0e1e8f7ce5..42c559c999 100644 --- a/src/main/resources/resource/WebGui/app/service/js/FiniteStateMachineGui.js +++ b/src/main/resources/resource/WebGui/app/service/js/FiniteStateMachineGui.js @@ -49,8 +49,8 @@ angular.module('mrlapp.service.FiniteStateMachineGui', []).controller('FiniteSta _self.updateState(data) $scope.$apply() break - case 'onNewState': - $scope.current = data + case 'onStateChange': + $scope.current = data.current $scope.$apply() break default: @@ -60,7 +60,7 @@ angular.module('mrlapp.service.FiniteStateMachineGui', []).controller('FiniteSta } - msg.subscribe("publishNewState") + msg.subscribe("publishStateChange") msg.subscribe(this) } ]) diff --git a/src/main/resources/resource/WebGui/app/service/js/WebXRGui.js b/src/main/resources/resource/WebGui/app/service/js/WebXRGui.js new file mode 100644 index 0000000000..52fd02c526 --- /dev/null +++ b/src/main/resources/resource/WebGui/app/service/js/WebXRGui.js @@ -0,0 +1,43 @@ +angular.module('mrlapp.service.WebXRGui', []).controller('WebXRGuiCtrl', ['$scope', 'mrl', function($scope, mrl) { + console.info('WebXRGuiCtrl') + var _self = this + var msg = this.msg + $scope.poses = {} + $scope.events = {} + $scope.jointAngles = {} + + this.updateState = function(service) { + $scope.service = service + } + + this.onMsg = function(inMsg) { + let data = inMsg.data[0] + switch (inMsg.method) { + case 'onState': + _self.updateState(data) + $scope.$apply() + break + case 'onPose': + $scope.poses[data.name] = data + $scope.$apply() + break + case 'onEvent': + $scope.events[data.uuid] = data + $scope.$apply() + break + case 'onJointAngles': + $scope.jointAngles = {...$scope.jointAngles, ...data} + $scope.$apply() + break + default: + console.error("ERROR - unhandled method " + $scope.name + " " + inMsg.method) + break + } + } + + msg.subscribe('publishPose') + msg.subscribe('publishEvent') + msg.subscribe('publishJointAngles') + msg.subscribe(this) +} +]) diff --git a/src/main/resources/resource/WebGui/app/service/views/WebXRGui.html b/src/main/resources/resource/WebGui/app/service/views/WebXRGui.html new file mode 100644 index 0000000000..ff09de61bc --- /dev/null +++ b/src/main/resources/resource/WebGui/app/service/views/WebXRGui.html @@ -0,0 +1,64 @@ +
+ +
Controllers
+ + + + + + + + + + + + + + + + + + + + + + + +
namexyzpitchrollyaw
{{pose.name}}{{pose.position.x.toFixed(2)}}{{pose.position.y.toFixed(2)}}{{pose.position.z.toFixed(2)}}{{pose.orientation.pitch.toFixed(2)}}{{pose.orientation.roll.toFixed(2)}}{{pose.orientation.yaw.toFixed(2)}}

+ +
Events
+ + + + + + + + + + + + + + + + + +
idhandednesstypevalue
{{event.id}}{{event.meta.handedness}}{{event.type}}{{event.value}}

+ +
Joint Angles
+ + + + + + + + + + + + + +
jointangle
{{key}}{{value.toFixed(2)}}
+
\ No newline at end of file diff --git a/src/main/resources/resource/WebXR.png b/src/main/resources/resource/WebXR.png new file mode 100644 index 0000000000..2432b77c92 Binary files /dev/null and b/src/main/resources/resource/WebXR.png differ diff --git a/src/test/java/org/myrobotlab/service/OpenCVTest.java b/src/test/java/org/myrobotlab/service/OpenCVTest.java index 458cfef8ad..c821a47c02 100644 --- a/src/test/java/org/myrobotlab/service/OpenCVTest.java +++ b/src/test/java/org/myrobotlab/service/OpenCVTest.java @@ -95,37 +95,6 @@ public static void setUpBeforeClass() throws Exception { long ts = System.currentTimeMillis(); cv = (OpenCV) Runtime.start("cv", "OpenCV"); - /* - * - * log. - * warn("========= OpenCVTest - setupbefore class - started cv {} ms =========" - * , System.currentTimeMillis()-ts ); ts = System.currentTimeMillis(); log. - * warn("========= OpenCVTest - setupbefore class - starting capture =========" - * ); cv.capture(TEST_LOCAL_FACE_FILE_JPEG); log. - * warn("========= OpenCVTest - setupbefore class - started capture {} ms =========" - * , System.currentTimeMillis()-ts ); ts = System.currentTimeMillis(); log. - * warn("========= OpenCVTest - setupbefore class - starting getFaceDetect =========" - * ); cv.getFaceDetect(120000);// two minute wait to load all libraries log. - * warn("========= OpenCVTest - setupbefore class - started getFaceDetect {} ms =========" - * , System.currentTimeMillis()-ts ); ts = System.currentTimeMillis(); log. - * warn("========= OpenCVTest - setupbefore class - starting getClassifications =========" - * ); cv.reset(); OpenCVFilter yoloFilter = cv.addFilter("yolo"); // - * cv.getClassifications(120000); cv.capture(TEST_LOCAL_FACE_FILE_JPEG); - * log. - * warn("========= OpenCVTest - setupbefore class - started getClassifications {} ms =========" - * , System.currentTimeMillis()-ts ); - * - * ts = System.currentTimeMillis(); log. - * warn("========= OpenCVTest - setupbefore class - starting getOpenCVData =========" - * ); - * - * cv.reset(); cv.capture(TEST_LOCAL_MP4); cv.getOpenCVData(); log. - * warn("========= OpenCVTest - setupbefore class - started getOpenCVData {} ms =========" - * , System.currentTimeMillis()-ts ); cv.disableAll(); // if (!isHeadless()) - * { - no longer needed I believe - SwingGui now handles it - * - * // } - */ } // FIXME - do the following test diff --git a/src/test/java/org/myrobotlab/service/WebGuiSocketTest.java b/src/test/java/org/myrobotlab/service/WebGuiSocketTest.java new file mode 100644 index 0000000000..5a57823b8b --- /dev/null +++ b/src/test/java/org/myrobotlab/service/WebGuiSocketTest.java @@ -0,0 +1,137 @@ +package org.myrobotlab.service; + +import static org.junit.Assert.assertEquals; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.myrobotlab.codec.CodecUtils; +import org.myrobotlab.framework.MRLListener; +import org.myrobotlab.framework.Service; +import org.myrobotlab.logging.LoggerFactory; +import org.slf4j.Logger; + +import okhttp3.OkHttpClient; +import okhttp3.Request; +import okhttp3.WebSocket; +import okhttp3.WebSocketListener; + +public class WebGuiSocketTest { + + protected final static Logger log = LoggerFactory.getLogger(WebGuiSocketTest.class); + + protected WebSocket webSocket; + protected WebSocketListener webSocketListener; + protected BlockingQueue msgQueue = new LinkedBlockingQueue<>(); + protected WebGui webgui2; + + @Before + public void setUp() { + webgui2 = (WebGui) Runtime.create("webgui2", "WebGui"); + webgui2.autoStartBrowser(false); + webgui2.setPort(8889); + webgui2.startService(); + + Service.sleep(3); + OkHttpClient okHttpClient = new OkHttpClient(); + Request request = new Request.Builder() + .url("ws://localhost:8889/api/messages?user=root&pwd=pwd&session_id=2309adf3dlkdk&id=webgui-client").build(); + webSocketListener = new WebSocketListener() { + @Override + public void onOpen(WebSocket webSocket, okhttp3.Response response) { + // WebSocket connection is established + log.info("onOpen"); + } + + @Override + public void onMessage(WebSocket webSocket, String msg) { + log.info("onMessage {}", msg); + msgQueue.add(msg); + } + + @Override + public void onFailure(WebSocket webSocket, Throwable t, okhttp3.Response response) { + // Handle WebSocket failure + log.info("onFailure"); + } + }; + + webSocket = okHttpClient.newWebSocket(request, webSocketListener); + } + + @After + public void teardown() { + webSocket.cancel(); + } + + @Test + public void testWebSocketConnection() throws InterruptedException { + // Use a CountDownLatch to wait for the WebSocket connection to be + // established + // CountDownLatch latch = new CountDownLatch(1); + // webSocket.listener().onOpen(webSocket, null); + // Wait for the connection to be established + // assertTrue("WebSocket connection timeout", latch.await(5, TimeUnit.SECONDS)); + + + // if sucessfully connected we'll get an + // 1. addListener from its runtime for describe + // 2. then a describe is sent with a parameter that describes the requesting platform + + String json = msgQueue.poll(5, TimeUnit.SECONDS); + LinkedHashMapmsg = (LinkedHashMap)CodecUtils.fromJson(json); + assertEquals("runtime", msg.get("name")); + assertEquals("addListener", msg.get("method")); + Object data = msg.get("data"); + List p0 = (List)msg.get("data"); + MRLListener listener = CodecUtils.fromJson((String)p0.get(0), MRLListener.class); + assertEquals("describe", listener.topicMethod); + assertEquals("onDescribe", listener.callbackMethod); + assertEquals(String.format("runtime@%s", webgui2.getId()), listener.callbackName); + + // the client can optionally do the same thing + // send an addListener for describe + // then send a describe + + String addListener = "{\n" + + " \"msgId\": 1690173331106,\n" + + " \"name\": \"runtime\",\n" + + " \"method\": \"addListener\",\n" + + " \"sender\": \"runtime@p1\",\n" + + " \"sendingMethod\": \"sendTo\",\n" + + " \"data\": [\n" + + " \"\\\"describe\\\"\",\n" + + " \"\\\"runtime@p1\\\"\"\n" + + " ],\n" + + " \"encoding\": \"json\"\n" + + "}"; + + // FIXME - make describe + // String describe = + + // assert describe info + + //.info(json); + log.info("here"); + + } + + @Test + public void testWebSocketMessage() throws InterruptedException { + // Use a CountDownLatch to wait for the WebSocket message +// CountDownLatch latch = new CountDownLatch(1); +// +// String expectedMessage = "Hello, WebSocket!"; +// webSocket.listener().onMessage(webSocket, expectedMessage); +// +// // Wait for the message to be received +// assertTrue("WebSocket message timeout", latch.await(5, TimeUnit.SECONDS)); + } + +}