QT UI: sounds (commaai#2078)

* move android into own dir * fix name * maybe this works? qt ui doesn't work on mac * fix that * pc sound works * fix pc build * lowercase * that needs to be real_arch * split into classes * fix typo in lib * Fix cycle alerts * Add qt multimedia libs to install scripts * Add ui/android folder * Fix android build * Raise exception if sound init fails * add missing return Co-authored-by: Willem Melching <willem.melching@gmail.com> Co-authored-by: Comma Device <device@comma.ai>
gaanthony · Sep 9, 2020 · acd1bde · acd1bde
1 parent 9eca642
commit acd1bde
Show file tree

Hide file tree

Showing 16 changed files with 178 additions and 100 deletions.
diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base
@@ -38,6 +38,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     python-dev \
     python-pip \
     qt5-default \
+    qtmultimedia5-dev \
     sudo \
     wget \
   && rm -rf /var/lib/apt/lists/*

diff --git a/SConstruct b/SConstruct
@@ -190,6 +190,7 @@ if arch in ["x86_64", "Darwin", "larch64"]:
       QT_BASE + "include/QtGui",
       QT_BASE + "include/QtCore",
       QT_BASE + "include/QtDBus",
+      QT_BASE + "include/QtMultimedia",
     ]
     qt_env["LINKFLAGS"] += ["-F" + QT_BASE + "lib"]
   else:
@@ -199,6 +200,7 @@ if arch in ["x86_64", "Darwin", "larch64"]:
       f"/usr/include/{real_arch}-linux-gnu/qt5/QtGui",
       f"/usr/include/{real_arch}-linux-gnu/qt5/QtCore",
       f"/usr/include/{real_arch}-linux-gnu/qt5/QtDBus",
+      f"/usr/include/{real_arch}-linux-gnu/qt5/QtMultimedia",
     ]
 
   qt_env.Tool('qt')

diff --git a/release/files_common b/release/files_common
@@ -350,6 +350,9 @@ selfdrive/ui/text/text.c
 selfdrive/ui/qt/*.cc
 selfdrive/ui/qt/*.hpp
 
+selfdrive/ui/android/*.cc
+selfdrive/ui/android/*.hpp
+
 selfdrive/camerad/SConscript
 selfdrive/camerad/main.cc
 selfdrive/camerad/bufs.h

diff --git a/selfdrive/debug/cycle_alerts.py b/selfdrive/debug/cycle_alerts.py
@@ -7,52 +7,63 @@
 import time
 
 import cereal.messaging as messaging
-from selfdrive.controls.lib.events import EVENTS, Alert
+from selfdrive.car.honda.interface import CarInterface
+from selfdrive.controls.lib.events import ET, EVENTS, Alert, Events
+from selfdrive.controls.lib.alertmanager import AlertManager
 
-def now_millis(): return time.time() * 1000
 
-ALERTS = [a for _, et in EVENTS.items() for _, a in et.items() if isinstance(a, Alert)]
+def cycle_alerts(duration=200, is_metric=False):
+  alerts = list(EVENTS.keys())
+  print(alerts)
 
-#from cereal import car
-#ALERTS = [a for a in ALERTS if a.audible_alert == car.CarControl.HUDControl.AudibleAlert.chimeWarningRepeat]
-
-default_alerts = sorted(ALERTS, key=lambda alert: (alert.alert_size, len(alert.alert_text_2)))
-
-def cycle_alerts(duration_millis, alerts=None):
-  if alerts is None:
-    alerts = default_alerts
+  CP = CarInterface.get_params("HONDA CIVIC 2016 TOURING")
+  sm = messaging.SubMaster(['thermal', 'health', 'frame', 'model', 'liveCalibration',
+                            'dMonitoringState', 'plan', 'pathPlan', 'liveLocationKalman'])
 
   controls_state = messaging.pub_sock('controlsState')
+  thermal = messaging.pub_sock('thermal')
 
   idx, last_alert_millis = 0, 0
   alert = alerts[0]
+
+  events = Events()
+  AM = AlertManager()
+
+  frame = 0
+
   while 1:
-    if (now_millis() - last_alert_millis) > duration_millis:
-      alert = alerts[idx]
+    if frame % duration == 0:
       idx = (idx + 1) % len(alerts)
-      last_alert_millis = now_millis()
-      print('sending {}'.format(str(alert)))
+      events.clear()
+      events.add(alerts[idx])
+
+
+    current_alert_types = [ET.PERMANENT, ET.USER_DISABLE, ET.IMMEDIATE_DISABLE,
+                           ET.SOFT_DISABLE, ET.PRE_ENABLE, ET.NO_ENTRY,
+                           ET.ENABLE, ET.WARNING]
+    a = events.create_alerts(current_alert_types, [CP, sm, is_metric])
+    AM.add_many(frame, a)
+    AM.process_alerts(frame)
 
     dat = messaging.new_message()
     dat.init('controlsState')
 
-    dat.controlsState.alertType = alert.alert_type
-    dat.controlsState.alertText1 = alert.alert_text_1
-    dat.controlsState.alertText2 = alert.alert_text_2
-    dat.controlsState.alertSize = alert.alert_size
-    #dat.controlsState.alertStatus = alert.alert_status
-    dat.controlsState.alertSound = alert.audible_alert
+    dat.controlsState.alertText1 = AM.alert_text_1
+    dat.controlsState.alertText2 = AM.alert_text_2
+    dat.controlsState.alertSize = AM.alert_size
+    dat.controlsState.alertStatus = AM.alert_status
+    dat.controlsState.alertBlinkingRate = AM.alert_rate
+    dat.controlsState.alertType = AM.alert_type
+    dat.controlsState.alertSound = AM.audible_alert
     controls_state.send(dat.to_bytes())
 
+    dat = messaging.new_message()
+    dat.init('thermal')
+    dat.thermal.started = True
+    thermal.send(dat.to_bytes())
+
+    frame += 1
     time.sleep(0.01)
 
 if __name__ == '__main__':
-  parser = argparse.ArgumentParser()
-  parser.add_argument('--duration', type=int, default=1000)
-  parser.add_argument('--alert-types', nargs='+')
-  args = parser.parse_args()
-  alerts = None
-  if args.alert_types:
-    alerts = [next(a for a in ALERTS if a.alert_type==alert_type) for alert_type in args.alert_types]
-
-  cycle_alerts(args.duration, alerts=alerts)
+  cycle_alerts()
diff --git a/selfdrive/ui/SConscript b/selfdrive/ui/SConscript
@@ -5,11 +5,10 @@ libs = [common, 'zmq', 'czmq', 'capnp', 'kj', 'm', cereal, messaging, gpucommon,
 
 
 if qt_env is None:
-  src += ['sound.cc']
   libs += ['EGL', 'GLESv3', 'gnustl_shared', 'log', 'utils', 'gui', 'hardware', 'ui', 'CB', 'gsl', 'adreno_utils', 'OpenSLES', 'cutils', 'uuid', 'OpenCL']
   linkflags = ['-Wl,-rpath=/system/lib64,-rpath=/system/comma/usr/lib']
 
-  src = ["android_ui.cc"] + src
+  src += ["android/ui.cc", "android/sl_sound.cc"]
   env.Program('_ui', src,
     LINKFLAGS=linkflags,
     LIBS=libs)
@@ -18,14 +17,14 @@ else:
   qt_libs = ["pthread"]
 
   if arch == "Darwin":
-    qt_env["FRAMEWORKS"] += ["QtWidgets", "QtGui", "QtCore", "QtDBus"]
+    qt_env["FRAMEWORKS"] += ["QtWidgets", "QtGui", "QtCore", "QtDBus", "QtMultimedia"]
   else:
-    qt_libs += ["Qt5Widgets", "Qt5Gui", "Qt5Core", "Qt5DBus"]
+    qt_libs += ["Qt5Widgets", "Qt5Gui", "Qt5Core", "Qt5DBus", "Qt5Multimedia"]
 
   if arch == "larch64":
     qt_libs += ["GLESv2"]
   else:
     qt_libs += ["GL"]
 
-  qt_src = ["qt/ui.cc", "qt/window.cc", "qt/settings.cc"] + src
+  qt_src = ["qt/ui.cc", "qt/window.cc", "qt/settings.cc", "qt/qt_sound.cc"] + src
   qt_env.Program("_ui", qt_src, LIBS=qt_libs + libs)
diff --git a/selfdrive/ui/sound.cc → selfdrive/ui/android/sl_sound.cc b/selfdrive/ui/sound.cc → selfdrive/ui/android/sl_sound.cc
@@ -1,35 +1,31 @@
-
-#include "sound.hpp"
 #include <math.h>
 #include <stdlib.h>
 #include <atomic>
 #include "common/swaglog.h"
 #include "common/timing.h"
 
+#include "android/sl_sound.hpp"
+
 #define LogOnError(func, msg) \
   if ((func) != SL_RESULT_SUCCESS) { LOGW(msg); }
 
 #define ReturnOnError(func, msg) \
   if ((func) != SL_RESULT_SUCCESS) { LOGW(msg); return false; }
 
-static std::map<AudibleAlert, std::pair<const char *, int>> sound_map {
-    {AudibleAlert::CHIME_DISENGAGE, {"../assets/sounds/disengaged.wav", 0}},
-    {AudibleAlert::CHIME_ENGAGE, {"../assets/sounds/engaged.wav", 0}},
-    {AudibleAlert::CHIME_WARNING1, {"../assets/sounds/warning_1.wav", 0}},
-    {AudibleAlert::CHIME_WARNING2, {"../assets/sounds/warning_2.wav", 0}},
-    {AudibleAlert::CHIME_WARNING2_REPEAT, {"../assets/sounds/warning_2.wav", 3}},
-    {AudibleAlert::CHIME_WARNING_REPEAT, {"../assets/sounds/warning_repeat.wav", 3}},
-    {AudibleAlert::CHIME_ERROR, {"../assets/sounds/error.wav", 0}},
-    {AudibleAlert::CHIME_PROMPT, {"../assets/sounds/error.wav", 0}}};
-
-struct Sound::Player {
+struct SLSound::Player {
   SLObjectItf player;
   SLPlayItf playItf;
   // slplay_callback runs on a background thread,use atomic to ensure thread safe.
   std::atomic<int> repeat;
 };
 
-bool Sound::init(int volume) {
+SLSound::SLSound() {
+  if (!init()){
+    throw std::runtime_error("Failed to initialize sound");
+  }
+}
+
+bool SLSound::init() {
   SLEngineOption engineOptions[] = {{SL_ENGINEOPTION_THREADSAFE, SL_BOOLEAN_TRUE}};
   const SLInterfaceID ids[1] = {SL_IID_VOLUME};
   const SLboolean req[1] = {SL_BOOLEAN_FALSE};
@@ -54,15 +50,13 @@ bool Sound::init(int volume) {
     ReturnOnError((*player)->GetInterface(player, SL_IID_PLAY, &playItf), "Failed to get player interface");
     ReturnOnError((*playItf)->SetPlayState(playItf, SL_PLAYSTATE_PAUSED), "Failed to initialize playstate to SL_PLAYSTATE_PAUSED");
 
-    player_[kv.first] = new Sound::Player{player, playItf};
+    player_[kv.first] = new SLSound::Player{player, playItf};
   }
-
-  setVolume(volume);
   return true;
 }
 
 void SLAPIENTRY slplay_callback(SLPlayItf playItf, void *context, SLuint32 event) {
-  Sound::Player *s = reinterpret_cast<Sound::Player *>(context);
+  SLSound::Player *s = reinterpret_cast<SLSound::Player *>(context);
   if (event == SL_PLAYEVENT_HEADATEND && s->repeat > 1) {
     --s->repeat;
     (*playItf)->SetPlayState(playItf, SL_PLAYSTATE_STOPPED);
@@ -71,7 +65,7 @@ void SLAPIENTRY slplay_callback(SLPlayItf playItf, void *context, SLuint32 event
   }
 }
 
-bool Sound::play(AudibleAlert alert) {
+bool SLSound::play(AudibleAlert alert) {
   if (currentSound_ != AudibleAlert::NONE) {
     stop();
   }
@@ -93,7 +87,7 @@ bool Sound::play(AudibleAlert alert) {
   return true;
 }
 
-void Sound::stop() {
+void SLSound::stop() {
   if (currentSound_ != AudibleAlert::NONE) {
     auto player = player_.at(currentSound_);
     player->repeat = 0;
@@ -102,9 +96,9 @@ void Sound::stop() {
   }
 }
 
-void Sound::setVolume(int volume) {
+void SLSound::setVolume(int volume) {
   if (last_volume_ == volume) return;
-  
+
   double current_time = nanos_since_boot();
   if ((current_time - last_set_volume_time_) > (5 * (1e+9))) { // 5s timeout on updating the volume
     char volume_change_cmd[64];
@@ -115,11 +109,15 @@ void Sound::setVolume(int volume) {
   }
 }
 
-Sound::~Sound() {
+SLSound::~SLSound() {
   for (auto &kv : player_) {
     (*(kv.second->player))->Destroy(kv.second->player);
     delete kv.second;
   }
-  if (outputMix_) (*outputMix_)->Destroy(outputMix_);
-  if (engine_) (*engine_)->Destroy(engine_);
+  if (outputMix_) {
+    (*outputMix_)->Destroy(outputMix_);
+  }
+  if (engine_) {
+    (*engine_)->Destroy(engine_);
+  }
 }
diff --git a/selfdrive/ui/android/sl_sound.hpp b/selfdrive/ui/android/sl_sound.hpp
@@ -0,0 +1,26 @@
+#pragma once
+#include <SLES/OpenSLES.h>
+#include <SLES/OpenSLES_Android.h>
+
+#include "sound.hpp"
+
+
+class SLSound : public Sound {
+public:
+  SLSound();
+  ~SLSound();
+  bool play(AudibleAlert alert);
+  void stop();
+  void setVolume(int volume);
+
+private:
+  bool init();
+  SLObjectItf engine_ = nullptr;
+  SLObjectItf outputMix_ = nullptr;
+  int last_volume_ = 0;
+  double last_set_volume_time_ = 0.;
+  AudibleAlert currentSound_ = AudibleAlert::NONE;
+  struct Player;
+  std::map<AudibleAlert, Player *> player_;
+  friend void SLAPIENTRY slplay_callback(SLPlayItf playItf, void *context, SLuint32 event);
+};
diff --git a/selfdrive/ui/android_ui.cc → selfdrive/ui/android/ui.cc b/selfdrive/ui/android_ui.cc → selfdrive/ui/android/ui.cc
@@ -12,6 +12,7 @@
 
 #include "ui.hpp"
 #include "paint.hpp"
+#include "android/sl_sound.hpp"
 
 // Includes for light sensor
 #include <cutils/properties.h>
@@ -169,10 +170,13 @@ int main(int argc, char* argv[]) {
   setpriority(PRIO_PROCESS, 0, -14);
 
   signal(SIGINT, (sighandler_t)set_do_exit);
+  SLSound sound;
 
   UIState uistate = {};
   UIState *s = &uistate;
   ui_init(s);
+  s->sound = &sound;
+
   set_awake(s, true);
   enable_event_processing(true);
 
@@ -201,7 +205,7 @@ int main(int argc, char* argv[]) {
 
   const int MIN_VOLUME = LEON ? 12 : 9;
   const int MAX_VOLUME = LEON ? 15 : 12;
-  assert(s->sound.init(MIN_VOLUME));
+  s->sound->setVolume(MIN_VOLUME);
 
   while (!do_exit) {
     if (!s->started || !s->vision_connected) {
@@ -238,7 +242,7 @@ int main(int argc, char* argv[]) {
     }
 
     // up one notch every 5 m/s
-    s->sound.setVolume(fmin(MAX_VOLUME, MIN_VOLUME + s->scene.controls_state.getVEgo() / 5));
+    s->sound->setVolume(fmin(MAX_VOLUME, MIN_VOLUME + s->scene.controls_state.getVEgo() / 5));
 
     // set brightness
     float clipped_brightness = fmin(512, (s->light_sensor*brightness_m) + brightness_b);

diff --git a/selfdrive/ui/qt/qt_sound.cc b/selfdrive/ui/qt/qt_sound.cc
@@ -0,0 +1,29 @@
+#include <QUrl>
+#include "qt/qt_sound.hpp"
+
+QtSound::QtSound() {
+  for (auto &kv : sound_map) {
+    auto path = QUrl::fromLocalFile(kv.second.first);
+    sounds[kv.first].setSource(path);
+  }
+}
+
+bool QtSound::play(AudibleAlert alert) {
+  sounds[alert].setLoopCount(sound_map[alert].second);
+  sounds[alert].play();
+  return true;
+}
+
+void QtSound::stop() {
+  for (auto &kv : sounds) {
+    kv.second.stop();
+  }
+}
+
+void QtSound::setVolume(int volume) {
+  // TODO: implement this
+}
+
+QtSound::~QtSound() {
+
+}
diff --git a/selfdrive/ui/qt/qt_sound.hpp b/selfdrive/ui/qt/qt_sound.hpp
@@ -0,0 +1,16 @@
+#pragma once
+
+#include <QSoundEffect>
+#include "sound.hpp"
+
+class QtSound : public Sound {
+public:
+  QtSound();
+  ~QtSound();
+  bool play(AudibleAlert alert);
+  void stop();
+  void setVolume(int volume);
+
+private:
+  std::map<AudibleAlert, QSoundEffect> sounds;
+};