diff --git a/Dockerfile.openpilot_base b/Dockerfile.openpilot_base index 25ac0826c8ed81..a5e9207b1e961c 100644 --- a/Dockerfile.openpilot_base +++ b/Dockerfile.openpilot_base @@ -38,6 +38,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ python-dev \ python-pip \ qt5-default \ + qtmultimedia5-dev \ sudo \ wget \ && rm -rf /var/lib/apt/lists/* diff --git a/SConstruct b/SConstruct index aef739e22e2b00..086b012dd4fc32 100644 --- a/SConstruct +++ b/SConstruct @@ -190,6 +190,7 @@ if arch in ["x86_64", "Darwin", "larch64"]: QT_BASE + "include/QtGui", QT_BASE + "include/QtCore", QT_BASE + "include/QtDBus", + QT_BASE + "include/QtMultimedia", ] qt_env["LINKFLAGS"] += ["-F" + QT_BASE + "lib"] else: @@ -199,6 +200,7 @@ if arch in ["x86_64", "Darwin", "larch64"]: f"/usr/include/{real_arch}-linux-gnu/qt5/QtGui", f"/usr/include/{real_arch}-linux-gnu/qt5/QtCore", f"/usr/include/{real_arch}-linux-gnu/qt5/QtDBus", + f"/usr/include/{real_arch}-linux-gnu/qt5/QtMultimedia", ] qt_env.Tool('qt') diff --git a/release/files_common b/release/files_common index 6238bb4e63f2eb..7f8920d023a716 100644 --- a/release/files_common +++ b/release/files_common @@ -350,6 +350,9 @@ selfdrive/ui/text/text.c selfdrive/ui/qt/*.cc selfdrive/ui/qt/*.hpp +selfdrive/ui/android/*.cc +selfdrive/ui/android/*.hpp + selfdrive/camerad/SConscript selfdrive/camerad/main.cc selfdrive/camerad/bufs.h diff --git a/selfdrive/debug/cycle_alerts.py b/selfdrive/debug/cycle_alerts.py index 63b7d758c5901f..17740e9657554c 100755 --- a/selfdrive/debug/cycle_alerts.py +++ b/selfdrive/debug/cycle_alerts.py @@ -7,52 +7,63 @@ import time import cereal.messaging as messaging -from selfdrive.controls.lib.events import EVENTS, Alert +from selfdrive.car.honda.interface import CarInterface +from selfdrive.controls.lib.events import ET, EVENTS, Alert, Events +from selfdrive.controls.lib.alertmanager import AlertManager -def now_millis(): return time.time() * 1000 -ALERTS = [a for _, et in EVENTS.items() for _, a in et.items() if isinstance(a, Alert)] +def cycle_alerts(duration=200, is_metric=False): + alerts = list(EVENTS.keys()) + print(alerts) -#from cereal import car -#ALERTS = [a for a in ALERTS if a.audible_alert == car.CarControl.HUDControl.AudibleAlert.chimeWarningRepeat] - -default_alerts = sorted(ALERTS, key=lambda alert: (alert.alert_size, len(alert.alert_text_2))) - -def cycle_alerts(duration_millis, alerts=None): - if alerts is None: - alerts = default_alerts + CP = CarInterface.get_params("HONDA CIVIC 2016 TOURING") + sm = messaging.SubMaster(['thermal', 'health', 'frame', 'model', 'liveCalibration', + 'dMonitoringState', 'plan', 'pathPlan', 'liveLocationKalman']) controls_state = messaging.pub_sock('controlsState') + thermal = messaging.pub_sock('thermal') idx, last_alert_millis = 0, 0 alert = alerts[0] + + events = Events() + AM = AlertManager() + + frame = 0 + while 1: - if (now_millis() - last_alert_millis) > duration_millis: - alert = alerts[idx] + if frame % duration == 0: idx = (idx + 1) % len(alerts) - last_alert_millis = now_millis() - print('sending {}'.format(str(alert))) + events.clear() + events.add(alerts[idx]) + + + current_alert_types = [ET.PERMANENT, ET.USER_DISABLE, ET.IMMEDIATE_DISABLE, + ET.SOFT_DISABLE, ET.PRE_ENABLE, ET.NO_ENTRY, + ET.ENABLE, ET.WARNING] + a = events.create_alerts(current_alert_types, [CP, sm, is_metric]) + AM.add_many(frame, a) + AM.process_alerts(frame) dat = messaging.new_message() dat.init('controlsState') - dat.controlsState.alertType = alert.alert_type - dat.controlsState.alertText1 = alert.alert_text_1 - dat.controlsState.alertText2 = alert.alert_text_2 - dat.controlsState.alertSize = alert.alert_size - #dat.controlsState.alertStatus = alert.alert_status - dat.controlsState.alertSound = alert.audible_alert + dat.controlsState.alertText1 = AM.alert_text_1 + dat.controlsState.alertText2 = AM.alert_text_2 + dat.controlsState.alertSize = AM.alert_size + dat.controlsState.alertStatus = AM.alert_status + dat.controlsState.alertBlinkingRate = AM.alert_rate + dat.controlsState.alertType = AM.alert_type + dat.controlsState.alertSound = AM.audible_alert controls_state.send(dat.to_bytes()) + dat = messaging.new_message() + dat.init('thermal') + dat.thermal.started = True + thermal.send(dat.to_bytes()) + + frame += 1 time.sleep(0.01) if __name__ == '__main__': - parser = argparse.ArgumentParser() - parser.add_argument('--duration', type=int, default=1000) - parser.add_argument('--alert-types', nargs='+') - args = parser.parse_args() - alerts = None - if args.alert_types: - alerts = [next(a for a in ALERTS if a.alert_type==alert_type) for alert_type in args.alert_types] - - cycle_alerts(args.duration, alerts=alerts) + cycle_alerts() diff --git a/selfdrive/ui/SConscript b/selfdrive/ui/SConscript index 17b945ce6d0761..ed8275ee6b4ff8 100644 --- a/selfdrive/ui/SConscript +++ b/selfdrive/ui/SConscript @@ -5,11 +5,10 @@ libs = [common, 'zmq', 'czmq', 'capnp', 'kj', 'm', cereal, messaging, gpucommon, if qt_env is None: - src += ['sound.cc'] libs += ['EGL', 'GLESv3', 'gnustl_shared', 'log', 'utils', 'gui', 'hardware', 'ui', 'CB', 'gsl', 'adreno_utils', 'OpenSLES', 'cutils', 'uuid', 'OpenCL'] linkflags = ['-Wl,-rpath=/system/lib64,-rpath=/system/comma/usr/lib'] - src = ["android_ui.cc"] + src + src += ["android/ui.cc", "android/sl_sound.cc"] env.Program('_ui', src, LINKFLAGS=linkflags, LIBS=libs) @@ -18,14 +17,14 @@ else: qt_libs = ["pthread"] if arch == "Darwin": - qt_env["FRAMEWORKS"] += ["QtWidgets", "QtGui", "QtCore", "QtDBus"] + qt_env["FRAMEWORKS"] += ["QtWidgets", "QtGui", "QtCore", "QtDBus", "QtMultimedia"] else: - qt_libs += ["Qt5Widgets", "Qt5Gui", "Qt5Core", "Qt5DBus"] + qt_libs += ["Qt5Widgets", "Qt5Gui", "Qt5Core", "Qt5DBus", "Qt5Multimedia"] if arch == "larch64": qt_libs += ["GLESv2"] else: qt_libs += ["GL"] - qt_src = ["qt/ui.cc", "qt/window.cc", "qt/settings.cc"] + src + qt_src = ["qt/ui.cc", "qt/window.cc", "qt/settings.cc", "qt/qt_sound.cc"] + src qt_env.Program("_ui", qt_src, LIBS=qt_libs + libs) diff --git a/selfdrive/ui/sound.cc b/selfdrive/ui/android/sl_sound.cc similarity index 79% rename from selfdrive/ui/sound.cc rename to selfdrive/ui/android/sl_sound.cc index 79a8fd6e261d59..95cebab9c4d4a3 100644 --- a/selfdrive/ui/sound.cc +++ b/selfdrive/ui/android/sl_sound.cc @@ -1,35 +1,31 @@ - -#include "sound.hpp" #include #include #include #include "common/swaglog.h" #include "common/timing.h" +#include "android/sl_sound.hpp" + #define LogOnError(func, msg) \ if ((func) != SL_RESULT_SUCCESS) { LOGW(msg); } #define ReturnOnError(func, msg) \ if ((func) != SL_RESULT_SUCCESS) { LOGW(msg); return false; } -static std::map> sound_map { - {AudibleAlert::CHIME_DISENGAGE, {"../assets/sounds/disengaged.wav", 0}}, - {AudibleAlert::CHIME_ENGAGE, {"../assets/sounds/engaged.wav", 0}}, - {AudibleAlert::CHIME_WARNING1, {"../assets/sounds/warning_1.wav", 0}}, - {AudibleAlert::CHIME_WARNING2, {"../assets/sounds/warning_2.wav", 0}}, - {AudibleAlert::CHIME_WARNING2_REPEAT, {"../assets/sounds/warning_2.wav", 3}}, - {AudibleAlert::CHIME_WARNING_REPEAT, {"../assets/sounds/warning_repeat.wav", 3}}, - {AudibleAlert::CHIME_ERROR, {"../assets/sounds/error.wav", 0}}, - {AudibleAlert::CHIME_PROMPT, {"../assets/sounds/error.wav", 0}}}; - -struct Sound::Player { +struct SLSound::Player { SLObjectItf player; SLPlayItf playItf; // slplay_callback runs on a background thread,use atomic to ensure thread safe. std::atomic repeat; }; -bool Sound::init(int volume) { +SLSound::SLSound() { + if (!init()){ + throw std::runtime_error("Failed to initialize sound"); + } +} + +bool SLSound::init() { SLEngineOption engineOptions[] = {{SL_ENGINEOPTION_THREADSAFE, SL_BOOLEAN_TRUE}}; const SLInterfaceID ids[1] = {SL_IID_VOLUME}; const SLboolean req[1] = {SL_BOOLEAN_FALSE}; @@ -54,15 +50,13 @@ bool Sound::init(int volume) { ReturnOnError((*player)->GetInterface(player, SL_IID_PLAY, &playItf), "Failed to get player interface"); ReturnOnError((*playItf)->SetPlayState(playItf, SL_PLAYSTATE_PAUSED), "Failed to initialize playstate to SL_PLAYSTATE_PAUSED"); - player_[kv.first] = new Sound::Player{player, playItf}; + player_[kv.first] = new SLSound::Player{player, playItf}; } - - setVolume(volume); return true; } void SLAPIENTRY slplay_callback(SLPlayItf playItf, void *context, SLuint32 event) { - Sound::Player *s = reinterpret_cast(context); + SLSound::Player *s = reinterpret_cast(context); if (event == SL_PLAYEVENT_HEADATEND && s->repeat > 1) { --s->repeat; (*playItf)->SetPlayState(playItf, SL_PLAYSTATE_STOPPED); @@ -71,7 +65,7 @@ void SLAPIENTRY slplay_callback(SLPlayItf playItf, void *context, SLuint32 event } } -bool Sound::play(AudibleAlert alert) { +bool SLSound::play(AudibleAlert alert) { if (currentSound_ != AudibleAlert::NONE) { stop(); } @@ -93,7 +87,7 @@ bool Sound::play(AudibleAlert alert) { return true; } -void Sound::stop() { +void SLSound::stop() { if (currentSound_ != AudibleAlert::NONE) { auto player = player_.at(currentSound_); player->repeat = 0; @@ -102,9 +96,9 @@ void Sound::stop() { } } -void Sound::setVolume(int volume) { +void SLSound::setVolume(int volume) { if (last_volume_ == volume) return; - + double current_time = nanos_since_boot(); if ((current_time - last_set_volume_time_) > (5 * (1e+9))) { // 5s timeout on updating the volume char volume_change_cmd[64]; @@ -115,11 +109,15 @@ void Sound::setVolume(int volume) { } } -Sound::~Sound() { +SLSound::~SLSound() { for (auto &kv : player_) { (*(kv.second->player))->Destroy(kv.second->player); delete kv.second; } - if (outputMix_) (*outputMix_)->Destroy(outputMix_); - if (engine_) (*engine_)->Destroy(engine_); + if (outputMix_) { + (*outputMix_)->Destroy(outputMix_); + } + if (engine_) { + (*engine_)->Destroy(engine_); + } } diff --git a/selfdrive/ui/android/sl_sound.hpp b/selfdrive/ui/android/sl_sound.hpp new file mode 100644 index 00000000000000..7925277b9d80d3 --- /dev/null +++ b/selfdrive/ui/android/sl_sound.hpp @@ -0,0 +1,26 @@ +#pragma once +#include +#include + +#include "sound.hpp" + + +class SLSound : public Sound { +public: + SLSound(); + ~SLSound(); + bool play(AudibleAlert alert); + void stop(); + void setVolume(int volume); + +private: + bool init(); + SLObjectItf engine_ = nullptr; + SLObjectItf outputMix_ = nullptr; + int last_volume_ = 0; + double last_set_volume_time_ = 0.; + AudibleAlert currentSound_ = AudibleAlert::NONE; + struct Player; + std::map player_; + friend void SLAPIENTRY slplay_callback(SLPlayItf playItf, void *context, SLuint32 event); +}; diff --git a/selfdrive/ui/android_ui.cc b/selfdrive/ui/android/ui.cc similarity index 97% rename from selfdrive/ui/android_ui.cc rename to selfdrive/ui/android/ui.cc index dd9fff93c9f4a8..06d11a94ec9934 100644 --- a/selfdrive/ui/android_ui.cc +++ b/selfdrive/ui/android/ui.cc @@ -12,6 +12,7 @@ #include "ui.hpp" #include "paint.hpp" +#include "android/sl_sound.hpp" // Includes for light sensor #include @@ -169,10 +170,13 @@ int main(int argc, char* argv[]) { setpriority(PRIO_PROCESS, 0, -14); signal(SIGINT, (sighandler_t)set_do_exit); + SLSound sound; UIState uistate = {}; UIState *s = &uistate; ui_init(s); + s->sound = &sound; + set_awake(s, true); enable_event_processing(true); @@ -201,7 +205,7 @@ int main(int argc, char* argv[]) { const int MIN_VOLUME = LEON ? 12 : 9; const int MAX_VOLUME = LEON ? 15 : 12; - assert(s->sound.init(MIN_VOLUME)); + s->sound->setVolume(MIN_VOLUME); while (!do_exit) { if (!s->started || !s->vision_connected) { @@ -238,7 +242,7 @@ int main(int argc, char* argv[]) { } // up one notch every 5 m/s - s->sound.setVolume(fmin(MAX_VOLUME, MIN_VOLUME + s->scene.controls_state.getVEgo() / 5)); + s->sound->setVolume(fmin(MAX_VOLUME, MIN_VOLUME + s->scene.controls_state.getVEgo() / 5)); // set brightness float clipped_brightness = fmin(512, (s->light_sensor*brightness_m) + brightness_b); diff --git a/selfdrive/ui/qt/qt_sound.cc b/selfdrive/ui/qt/qt_sound.cc new file mode 100644 index 00000000000000..c86794b1643af8 --- /dev/null +++ b/selfdrive/ui/qt/qt_sound.cc @@ -0,0 +1,29 @@ +#include +#include "qt/qt_sound.hpp" + +QtSound::QtSound() { + for (auto &kv : sound_map) { + auto path = QUrl::fromLocalFile(kv.second.first); + sounds[kv.first].setSource(path); + } +} + +bool QtSound::play(AudibleAlert alert) { + sounds[alert].setLoopCount(sound_map[alert].second); + sounds[alert].play(); + return true; +} + +void QtSound::stop() { + for (auto &kv : sounds) { + kv.second.stop(); + } +} + +void QtSound::setVolume(int volume) { + // TODO: implement this +} + +QtSound::~QtSound() { + +} diff --git a/selfdrive/ui/qt/qt_sound.hpp b/selfdrive/ui/qt/qt_sound.hpp new file mode 100644 index 00000000000000..c2aab2de446ff3 --- /dev/null +++ b/selfdrive/ui/qt/qt_sound.hpp @@ -0,0 +1,16 @@ +#pragma once + +#include +#include "sound.hpp" + +class QtSound : public Sound { +public: + QtSound(); + ~QtSound(); + bool play(AudibleAlert alert); + void stop(); + void setVolume(int volume); + +private: + std::map sounds; +}; diff --git a/selfdrive/ui/qt/window.cc b/selfdrive/ui/qt/window.cc index d3b73952a75b85..54f3ef5dfd3bbb 100644 --- a/selfdrive/ui/qt/window.cc +++ b/selfdrive/ui/qt/window.cc @@ -13,7 +13,6 @@ #include "settings.hpp" #include "paint.hpp" -#include "sound.hpp" volatile sig_atomic_t do_exit = 0; @@ -40,11 +39,11 @@ MainWindow::MainWindow(QWidget *parent) : QWidget(parent) { )"); } -void MainWindow::openSettings(){ +void MainWindow::openSettings() { main_layout->setCurrentIndex(1); } -void MainWindow::closeSettings(){ +void MainWindow::closeSettings() { main_layout->setCurrentIndex(0); } @@ -52,6 +51,7 @@ void MainWindow::closeSettings(){ GLWindow::GLWindow(QWidget *parent) : QOpenGLWidget(parent) { timer = new QTimer(this); QObject::connect(timer, SIGNAL(timeout()), this, SLOT(timerUpdate())); + } GLWindow::~GLWindow() { @@ -68,6 +68,7 @@ void GLWindow::initializeGL() { ui_state = new UIState(); ui_init(ui_state); + ui_state->sound = &sound; ui_state->fb_w = vwp_w; ui_state->fb_h = vwp_h; @@ -103,13 +104,6 @@ void GLWindow::mousePressEvent(QMouseEvent *e) { } -/* HACKS */ -bool Sound::init(int volume) { return true; } -bool Sound::play(AudibleAlert alert) { printf("play sound: %d\n", (int)alert); return true; } -void Sound::stop() {} -void Sound::setVolume(int volume) {} -Sound::~Sound() {} - GLuint visionimg_to_gl(const VisionImg *img, EGLImageKHR *pkhr, void **pph) { unsigned int texture; glGenTextures(1, &texture); diff --git a/selfdrive/ui/qt/window.hpp b/selfdrive/ui/qt/window.hpp index 927c3fa53cc620..c8950bd5687940 100644 --- a/selfdrive/ui/qt/window.hpp +++ b/selfdrive/ui/qt/window.hpp @@ -7,6 +7,7 @@ #include #include +#include "qt/qt_sound.hpp" #include "ui/ui.hpp" class MainWindow : public QWidget @@ -45,6 +46,7 @@ class GLWindow : public QOpenGLWidget, protected QOpenGLFunctions private: QTimer * timer; UIState * ui_state; + QtSound sound; public slots: void timerUpdate(); diff --git a/selfdrive/ui/sound.hpp b/selfdrive/ui/sound.hpp index d565f846ffcf46..93157830661376 100644 --- a/selfdrive/ui/sound.hpp +++ b/selfdrive/ui/sound.hpp @@ -2,31 +2,23 @@ #include #include "cereal/gen/cpp/log.capnp.h" -#if defined(QCOM) -#include -#include -#endif - typedef cereal::CarControl::HUDControl::AudibleAlert AudibleAlert; -class Sound { - public: - Sound() = default; - bool init(int volume); - bool play(AudibleAlert alert); - void stop(); - void setVolume(int volume); - ~Sound(); +static std::map> sound_map { + // AudibleAlert, (file path, loop count) + {AudibleAlert::CHIME_DISENGAGE, {"../assets/sounds/disengaged.wav", 0}}, + {AudibleAlert::CHIME_ENGAGE, {"../assets/sounds/engaged.wav", 0}}, + {AudibleAlert::CHIME_WARNING1, {"../assets/sounds/warning_1.wav", 0}}, + {AudibleAlert::CHIME_WARNING2, {"../assets/sounds/warning_2.wav", 0}}, + {AudibleAlert::CHIME_WARNING2_REPEAT, {"../assets/sounds/warning_2.wav", 3}}, + {AudibleAlert::CHIME_WARNING_REPEAT, {"../assets/sounds/warning_repeat.wav", 3}}, + {AudibleAlert::CHIME_ERROR, {"../assets/sounds/error.wav", 0}}, + {AudibleAlert::CHIME_PROMPT, {"../assets/sounds/error.wav", 0}} +}; -#if defined(QCOM) - private: - SLObjectItf engine_ = nullptr; - SLObjectItf outputMix_ = nullptr; - int last_volume_ = 0; - double last_set_volume_time_ = 0.; - AudibleAlert currentSound_ = AudibleAlert::NONE; - struct Player; - std::map player_; - friend void SLAPIENTRY slplay_callback(SLPlayItf playItf, void *context, SLuint32 event); -#endif +class Sound { +public: + virtual bool play(AudibleAlert alert) = 0; + virtual void stop() = 0; + virtual void setVolume(int volume) = 0; }; diff --git a/selfdrive/ui/ui.cc b/selfdrive/ui/ui.cc index 59dec1b742dcc9..ddf320e5303d4b 100644 --- a/selfdrive/ui/ui.cc +++ b/selfdrive/ui/ui.cc @@ -130,9 +130,9 @@ void update_sockets(UIState *s) { auto alert_sound = scene.controls_state.getAlertSound(); if (scene.alert_type.compare(scene.controls_state.getAlertType()) != 0) { if (alert_sound == AudibleAlert::NONE) { - s->sound.stop(); + s->sound->stop(); } else { - s->sound.play(alert_sound); + s->sound->play(alert_sound); } } scene.alert_text1 = scene.controls_state.getAlertText1(); @@ -254,7 +254,7 @@ void ui_update(UIState *s) { } else { // car is started, but controls is lagging or died if (s->scene.alert_text2 != "Controls Unresponsive") { - s->sound.play(AudibleAlert::CHIME_WARNING_REPEAT); + s->sound->play(AudibleAlert::CHIME_WARNING_REPEAT); LOGE("Controls unresponsive"); } diff --git a/selfdrive/ui/ui.hpp b/selfdrive/ui/ui.hpp index 6c1c1b4ab91229..6082991539a595 100644 --- a/selfdrive/ui/ui.hpp +++ b/selfdrive/ui/ui.hpp @@ -174,7 +174,7 @@ typedef struct UIState { SubMaster *sm; - Sound sound; + Sound *sound; UIStatus status; UIScene scene; cereal::UiLayoutState::App active_app; diff --git a/tools/ubuntu_setup.sh b/tools/ubuntu_setup.sh index 94c0850da12fa2..a77a1123bdf9b8 100755 --- a/tools/ubuntu_setup.sh +++ b/tools/ubuntu_setup.sh @@ -43,6 +43,7 @@ sudo apt-get update && sudo apt-get install -y \ python-dev \ python-pip \ qt5-default \ + qtmultimedia5-dev \ screen \ sudo \ vim \