Voice entry demo

.gitmodules

@@ -6,7 +6,7 @@
   url = ../../commaai/opendbc.git
 [submodule "cereal"]
   path = cereal
-  url = ../../commaai/cereal.git
+  url = ../../jakethesnake420/cereal.git
 [submodule "rednose_repo"]
   path = rednose_repo
   url = ../../commaai/rednose.git

common/params.cc

@@ -187,6 +187,7 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"RecordFrontLock", PERSISTENT},  // for the internal fleet
     {"ReplayControlsState", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
     {"SnoozeUpdate", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"SpeechToTextAllowed", PERSISTENT},
     {"SshEnabled", PERSISTENT},
     {"TermsVersion", PERSISTENT},
     {"Timezone", PERSISTENT},
@@ -205,6 +206,7 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"UpdaterLastFetchTime", PERSISTENT},
     {"Version", PERSISTENT},
     {"VisionRadarToggle", PERSISTENT},
+    {"WakeWordDetected", CLEAR_ON_MANAGER_START},
     {"WheeledBody", PERSISTENT},
 };
 

pyproject.toml

@@ -105,6 +105,7 @@ sounddevice = "*"
 spidev = { version = "*", platform = "linux" }
 sympy = "*"
 websocket_client = "*"
+rev_ai = { git = "https://github.com/jakethesnake420/revai-python-sdk.git", branch = "patch-1" }
 
 # these should be removed
 markdown-it-py = "*"

selfdrive/manager/process_config.py

@@ -48,7 +48,7 @@ def only_offroad(started, params, CP: car.CarParams) -> bool:
   NativeProcess("logcatd", "system/logcatd", ["./logcatd"], only_onroad),
   NativeProcess("proclogd", "system/proclogd", ["./proclogd"], only_onroad),
   PythonProcess("logmessaged", "system.logmessaged", always_run),
-  PythonProcess("micd", "system.micd", iscar),
+  PythonProcess("micd", "system.micd", always_run),
   PythonProcess("timezoned", "system.timezoned", always_run, enabled=not PC),
 
   PythonProcess("dmonitoringmodeld", "selfdrive.modeld.dmonitoringmodeld", driverview, enabled=(not PC or WEBCAM)),
@@ -81,6 +81,8 @@ def only_offroad(started, params, CP: car.CarParams) -> bool:
   PythonProcess("updated", "selfdrive.updated", only_offroad, enabled=not PC),
   PythonProcess("uploader", "system.loggerd.uploader", always_run),
   PythonProcess("statsd", "selfdrive.statsd", always_run),
+  PythonProcess("speechd", "system.assistant.rev_speechd", always_run),
+  PythonProcess("wakewordd", "system.assistant.wakewordd", always_run),
 
   # debug procs
   NativeProcess("bridge", "cereal/messaging", ["./bridge"], notcar),

selfdrive/ui/SConscript

@@ -27,7 +27,7 @@ widgets_src = ["ui.cc", "qt/widgets/input.cc", "qt/widgets/wifi.cc",
                "qt/widgets/ssh_keys.cc", "qt/widgets/toggle.cc", "qt/widgets/controls.cc",
                "qt/widgets/offroad_alerts.cc", "qt/widgets/prime.cc", "qt/widgets/keyboard.cc",
                "qt/widgets/scrollview.cc", "qt/widgets/cameraview.cc", "#third_party/qrcode/QrCode.cc",
-               "qt/request_repeater.cc", "qt/qt_window.cc", "qt/network/networking.cc", "qt/network/wifi_manager.cc"]
+               "qt/request_repeater.cc", "qt/qt_window.cc", "qt/network/networking.cc", "qt/network/wifi_manager.cc", "qt/widgets/assistant.cc"]
 
 qt_env['CPPDEFINES'] = []
 if maps:

selfdrive/ui/qt/home.cc

@@ -65,6 +65,7 @@ void HomeWindow::updateState(const UIState &s) {
     body->setEnabled(true);
     slayout->setCurrentWidget(body);
   }
+  emit requestRaiseAssistantOverlay();
 }
 
 void HomeWindow::offroadTransition(bool offroad) {

selfdrive/ui/qt/home.h

@@ -50,6 +50,7 @@ class HomeWindow : public QWidget {
 signals:
   void openSettings(int index = 0, const QString &param = "");
   void closeSettings();
+  void requestRaiseAssistantOverlay();
 
 public slots:
   void offroadTransition(bool offroad);

selfdrive/ui/qt/widgets/assistant.cc

@@ -0,0 +1,94 @@
+#include "assistant.h"
+
+AssistantOverlay::AssistantOverlay(QWidget *parent) : QLabel(parent) {
+
+  setStyleSheet("QLabel {"
+                "  background-color: #373737;"
+                "  border-radius: 20px;"
+                "  font-family: 'Inter';"
+                "  font-size: 60px;"
+                "  color: white;"  // Text color
+                "}");
+
+  // Set up the animations
+  showAnimation = new QPropertyAnimation(this, "geometry");
+  showAnimation->setDuration(250); // Duration in milliseconds
+
+  hideAnimation = new QPropertyAnimation(this, "geometry");
+  hideAnimation->setDuration(250);
+
+  int height = 100; // Fixed height
+  setGeometry(0, 0, 0, height);
+
+  hideTimer = new QTimer(this);
+  connect(hideTimer, &QTimer::timeout, this, &AssistantOverlay::animateHide);
+
+  QObject::connect(uiState(), &UIState::uiUpdate, this, &AssistantOverlay::updateState);
+  hide();
+}
+
+void AssistantOverlay::animateShow() {
+  parentCenterX = parentWidget()->width() / 2;
+  finalWidth = parentWidget()->width() * 0.5;
+  startX = parentCenterX - finalWidth / 2;
+  QRect startRect(parentCenterX, 0, 0, height()); // Centered, zero width
+  QRect endRect(startX, 0, finalWidth, height()); // Adjusted x, final width
+  showAnimation->setStartValue(startRect);
+  showAnimation->setEndValue(endRect);
+  show();
+  showAnimation->start();
+}
+
+void AssistantOverlay::animateHide() {
+  parentCenterX = parentWidget()->width() / 2;
+  finalWidth = parentWidget()->width() * 0.5;
+  startX = parentCenterX - finalWidth / 2;
+  QRect startRect(startX, 0, finalWidth, height()); // Adjusted x, final width
+  QRect endRect(parentCenterX, 0, 0, height()); // Centered, zero width
+  hideAnimation->setStartValue(startRect);
+  hideAnimation->setEndValue(endRect);
+  hideAnimation->start();
+  hideTimer->stop();
+}
+
+void AssistantOverlay::updateText(QString text) {
+  this->setText(text);
+  this->setAlignment(QFontMetrics(this->font()).horizontalAdvance(text) > this->finalWidth ? Qt::AlignRight : Qt::AlignCenter);
+}
+
+void AssistantOverlay::updateState(const UIState &s) {
+  const SubMaster &sm = *(s.sm);
+  static bool show_allowed = false;
+  static bool visable = false;
+  if (!sm.updated("speechToText")) {
+    return; // Early exit if speechToText is not updated
+  }
+  // Should probably refactor to a switch statement but its working.
+  if (cereal::SpeechToText::State::BEGIN == sm["speechToText"].getSpeechToText().getState()) {
+    show_allowed = true;
+    this->animateShow();
+    updateText("Hello, I'm listening");
+    if (hideTimer->isActive()) {
+      hideTimer->stop();
+    }
+    visable = true; // Require begin state or not valid to show
+  } else if (!sm["speechToText"].getValid()){ // show if not valid and show set the error text, then lock out and hide until next begin
+    if (!visable && show_allowed) {this->animateShow(); visable = true;}
+    updateText("Sorry, an error occorred");
+    hideTimer->start(8000);
+    show_allowed = false;
+  } else if (cereal::SpeechToText::State::EMPTY == sm["speechToText"].getSpeechToText().getState()){
+    updateText("Sorry, I didn't catch that");
+    hideTimer->start(8000);
+    visable = false;
+    show_allowed = false;
+  } else if (show_allowed){ // Interim and Final Results
+    updateText(QString::fromStdString(sm["speechToText"].getSpeechToText().getResult()));
+    if (sm["speechToText"].getSpeechToText().getFinalResultReady()) {
+      hideTimer->start(8000);
+      visable = false;
+    }
+  } else { //shouldn't get here unless I missed something
+    qWarning() << "AssistantOverlay in bad state";
+  }
+}

selfdrive/ui/qt/widgets/assistant.h

@@ -0,0 +1,28 @@
+#pragma once
+
+#include "selfdrive/ui/ui.h"
+#include <QLabel>
+#include <QPropertyAnimation>
+
+class AssistantOverlay : public QLabel {
+  Q_OBJECT
+
+public:
+  explicit AssistantOverlay(QWidget *parent = nullptr);
+  void animateShow();
+  void animateHide();
+
+private:
+  void updateText(const QString text);
+  void startHideTimer();
+  QTimer *hideTimer;
+  QPropertyAnimation *showAnimation;
+  QPropertyAnimation *hideAnimation;
+  int parentCenterX;
+  int finalWidth;
+  int startX;
+
+private slots:
+  void updateState(const UIState &s);
+
+};

selfdrive/ui/qt/window.cc

@@ -8,10 +8,12 @@ MainWindow::MainWindow(QWidget *parent) : QWidget(parent) {
   main_layout = new QStackedLayout(this);
   main_layout->setMargin(0);
 
+  assistantOverlay = new AssistantOverlay(this);
   homeWindow = new HomeWindow(this);
   main_layout->addWidget(homeWindow);
   QObject::connect(homeWindow, &HomeWindow::openSettings, this, &MainWindow::openSettings);
   QObject::connect(homeWindow, &HomeWindow::closeSettings, this, &MainWindow::closeSettings);
+  QObject::connect(homeWindow, &HomeWindow::requestRaiseAssistantOverlay, this, &MainWindow::raiseAssistantOverlay);
 
   settingsWindow = new SettingsWindow(this);
   main_layout->addWidget(settingsWindow);
@@ -33,15 +35,18 @@ MainWindow::MainWindow(QWidget *parent) : QWidget(parent) {
     main_layout->setCurrentWidget(onboardingWindow);
   }
 
+
   QObject::connect(uiState(), &UIState::offroadTransition, [=](bool offroad) {
     if (!offroad) {
       closeSettings();
     }
+    assistantOverlay->raise();
   });
   QObject::connect(device(), &Device::interactiveTimeout, [=]() {
     if (main_layout->currentWidget() == settingsWindow) {
       closeSettings();
     }
+    assistantOverlay->raise();
   });
 
   // load fonts
@@ -68,6 +73,7 @@ MainWindow::MainWindow(QWidget *parent) : QWidget(parent) {
 void MainWindow::openSettings(int index, const QString &param) {
   main_layout->setCurrentWidget(settingsWindow);
   settingsWindow->setCurrentPanel(index, param);
+  assistantOverlay->raise();
 }
 
 void MainWindow::closeSettings() {
@@ -80,6 +86,7 @@ void MainWindow::closeSettings() {
       homeWindow->showMapPanel(true);
     }
   }
+  assistantOverlay->raise();
 }
 
 bool MainWindow::eventFilter(QObject *obj, QEvent *event) {

selfdrive/ui/qt/window.h

@@ -6,20 +6,23 @@
 #include "selfdrive/ui/qt/home.h"
 #include "selfdrive/ui/qt/offroad/onboarding.h"
 #include "selfdrive/ui/qt/offroad/settings.h"
+#include "selfdrive/ui/qt/widgets/assistant.h"
 
 class MainWindow : public QWidget {
   Q_OBJECT
 
 public:
   explicit MainWindow(QWidget *parent = 0);
-
+  
 private:
   bool eventFilter(QObject *obj, QEvent *event) override;
   void openSettings(int index = 0, const QString &param = "");
   void closeSettings();
+  void raiseAssistantOverlay() {assistantOverlay->raise();};
 
   QStackedLayout *main_layout;
   HomeWindow *homeWindow;
   SettingsWindow *settingsWindow;
   OnboardingWindow *onboardingWindow;
+  AssistantOverlay *assistantOverlay;
 };

selfdrive/ui/ui.cc

@@ -248,7 +248,7 @@ UIState::UIState(QObject *parent) : QObject(parent) {
   sm = std::make_unique<SubMaster, const std::initializer_list<const char *>>({
     "modelV2", "controlsState", "liveCalibration", "radarState", "deviceState",
     "pandaStates", "carParams", "driverMonitoringState", "carState", "liveLocationKalman", "driverStateV2",
-    "wideRoadCameraState", "managerState", "navInstruction", "navRoute", "uiPlan",
+    "wideRoadCameraState", "managerState", "navInstruction", "navRoute", "uiPlan", "speechToText",
   });
 
   Params params;

system/assistant/openwakeword/README.md

@@ -0,0 +1,19 @@
+The openwakeword driectory code was copied from https://github.com/dscripka/openWakeWord 
+and then stripped down to the essentials for Openpilot's purposes.
+
+To test wake word detection on the comma device or PC, run wakeword.py and say "alexa".
+Make sure you have onnxruntime==1.16.3 when running on the comma device.
+
+pip install onnxruntime==1.16.3
+
+You can also run rev_speechd.py which will wait for the "WakeWordDetected" param to be set.
+To setup the Rev.Ai api you need to install rev_ai:
+
+pip install rev_ai
+
+You also need to set your rev ai acccess token which can be obtained with a free account. https://www.rev.ai/access-token
+Once you have your token you can paste it in launch_openpilot.sh.
+
+export REVAI_ACCESS_TOKEN=""
+
+Once you have everything set up you can run ./launch_openpilot and see the assistant overlay on the UI. You can also run ./ui, rev_speechd.py, wakeword.py, micd.py in their own terminals for testing.