Voice entry demo

.gitmodules

@@ -6,7 +6,7 @@
   url = ../../commaai/opendbc.git
 [submodule "cereal"]
   path = cereal
-  url = ../../commaai/cereal.git
+  url = ../../jakethesnake420/cereal.git
 [submodule "rednose_repo"]
   path = rednose_repo
   url = ../../commaai/rednose.git

common/params.cc

@@ -187,6 +187,7 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"RecordFrontLock", PERSISTENT},  // for the internal fleet
     {"ReplayControlsState", CLEAR_ON_MANAGER_START | CLEAR_ON_ONROAD_TRANSITION},
     {"SnoozeUpdate", CLEAR_ON_MANAGER_START | CLEAR_ON_OFFROAD_TRANSITION},
+    {"SpeechToTextAllowed", PERSISTENT},
     {"SshEnabled", PERSISTENT},
     {"TermsVersion", PERSISTENT},
     {"Timezone", PERSISTENT},
@@ -205,6 +206,7 @@ std::unordered_map<std::string, uint32_t> keys = {
     {"UpdaterLastFetchTime", PERSISTENT},
     {"Version", PERSISTENT},
     {"VisionRadarToggle", PERSISTENT},
+    {"WakeWordDetected", CLEAR_ON_MANAGER_START},
     {"WheeledBody", PERSISTENT},
 };
 

pyproject.toml

@@ -105,6 +105,7 @@ sounddevice = "*"
 spidev = { version = "*", platform = "linux" }
 sympy = "*"
 websocket_client = "*"
+rev_ai = { git = "https://github.com/jakethesnake420/revai-python-sdk.git", branch = "patch-1" }
 
 # these should be removed
 markdown-it-py = "*"

selfdrive/manager/process_config.py

@@ -48,7 +48,7 @@ def only_offroad(started, params, CP: car.CarParams) -> bool:
   NativeProcess("logcatd", "system/logcatd", ["./logcatd"], only_onroad),
   NativeProcess("proclogd", "system/proclogd", ["./proclogd"], only_onroad),
   PythonProcess("logmessaged", "system.logmessaged", always_run),
-  PythonProcess("micd", "system.micd", iscar),
+  PythonProcess("micd", "system.micd", only_onroad if TICI else always_run),
   PythonProcess("timezoned", "system.timezoned", always_run, enabled=not PC),
 
   PythonProcess("dmonitoringmodeld", "selfdrive.modeld.dmonitoringmodeld", driverview, enabled=(not PC or WEBCAM)),
@@ -81,6 +81,8 @@ def only_offroad(started, params, CP: car.CarParams) -> bool:
   PythonProcess("updated", "selfdrive.updated", only_offroad, enabled=not PC),
   PythonProcess("uploader", "system.loggerd.uploader", always_run),
   PythonProcess("statsd", "selfdrive.statsd", always_run),
+  PythonProcess("speechd", "system.assistant.rev_speechd", only_onroad if TICI else always_run),
+  PythonProcess("wakewordd", "system.assistant.wakewordd", only_onroad if TICI else always_run),
 
   # debug procs
   NativeProcess("bridge", "cereal/messaging", ["./bridge"], notcar),

selfdrive/ui/SConscript

@@ -27,7 +27,7 @@ widgets_src = ["ui.cc", "qt/widgets/input.cc", "qt/widgets/wifi.cc",
                "qt/widgets/ssh_keys.cc", "qt/widgets/toggle.cc", "qt/widgets/controls.cc",
                "qt/widgets/offroad_alerts.cc", "qt/widgets/prime.cc", "qt/widgets/keyboard.cc",
                "qt/widgets/scrollview.cc", "qt/widgets/cameraview.cc", "#third_party/qrcode/QrCode.cc",
-               "qt/request_repeater.cc", "qt/qt_window.cc", "qt/network/networking.cc", "qt/network/wifi_manager.cc"]
+               "qt/request_repeater.cc", "qt/qt_window.cc", "qt/network/networking.cc", "qt/network/wifi_manager.cc", "qt/widgets/assistant.cc"]
 
 qt_env['CPPDEFINES'] = []
 if maps:

selfdrive/ui/qt/home.cc

@@ -65,6 +65,7 @@ void HomeWindow::updateState(const UIState &s) {
     body->setEnabled(true);
     slayout->setCurrentWidget(body);
   }
+  emit requestRaiseAssistantOverlay();
 }
 
 void HomeWindow::offroadTransition(bool offroad) {

selfdrive/ui/qt/home.h

@@ -50,6 +50,7 @@ class HomeWindow : public QWidget {
 signals:
   void openSettings(int index = 0, const QString &param = "");
   void closeSettings();
+  void requestRaiseAssistantOverlay();
 
 public slots:
   void offroadTransition(bool offroad);

selfdrive/ui/qt/widgets/assistant.cc

@@ -0,0 +1,88 @@
+#include "selfdrive/ui/qt/widgets/assistant.h"
+
+AssistantOverlay::AssistantOverlay(QWidget *parent) : QLabel(parent) {
+
+  setStyleSheet("QLabel {"
+                "  background-color: #373737;"
+                "  border-radius: 20px;"
+                "  font-family: 'Inter';"
+                "  font-size: 60px;"
+                "  color: white;"  // Text color
+                "}");
+
+  // Set up the animations
+  showAnimation = new QPropertyAnimation(this, "geometry");
+  showAnimation->setDuration(250); // Duration in milliseconds
+  hideAnimation = new QPropertyAnimation(this, "geometry");
+  hideAnimation->setDuration(250);
+  int height = 100; // Fixed height
+  setGeometry(0, 0, 0, height);
+  hide();
+
+  hideTimer = new QTimer(this);
+  connect(hideTimer, &QTimer::timeout, this, [this]() { animateOverlay(false); });
+  QObject::connect(uiState(), &UIState::uiUpdate, this, &AssistantOverlay::updateState);
+}
+
+void AssistantOverlay::animateOverlay(bool show) {
+  int parentCenterX = parentWidget()->width() / 2;
+  finalWidth = parentWidget()->width() * 0.5;
+  int startX = parentCenterX - finalWidth / 2;
+  QRect centerRect(parentCenterX, 0, 0, height()); // Centered, zero width
+  QRect fullRect(startX, 0, finalWidth, height()); // Adjusted x, final width
+
+  if (show) {
+    showAnimation->setStartValue(centerRect);
+    showAnimation->setEndValue(fullRect);
+    this->show();
+    showAnimation->start();
+  } else {
+    hideAnimation->setStartValue(fullRect);
+    hideAnimation->setEndValue(centerRect);
+    hideAnimation->start();
+    hideTimer->stop();
+  }
+}
+
+void AssistantOverlay::updateText(QString text) {
+  this->setText(text);
+  this->setAlignment(QFontMetrics(this->font()).horizontalAdvance(text) > this->finalWidth ? Qt::AlignRight : Qt::AlignCenter);
+}
+
+void AssistantOverlay::updateState(const UIState &s) {
+  const SubMaster &sm = *(s.sm);
+  if (!sm.updated("speechToText")) return;
+
+  static cereal::SpeechToText::State current_state = cereal::SpeechToText::State::NONE;
+  cereal::SpeechToText::State request_state = sm["speechToText"].getSpeechToText().getState();
+  // Check for valid state transition
+  if (current_state == cereal::SpeechToText::State::BEGIN  ||
+     (current_state == cereal::SpeechToText::State::NONE   &&
+     (request_state == cereal::SpeechToText::State::EMPTY  ||
+      request_state == cereal::SpeechToText::State::FINAL  ||
+      request_state == cereal::SpeechToText::State::NONE)) ||
+      request_state == cereal::SpeechToText::State::BEGIN) {
+
+    current_state = request_state;  // Update state
+    switch (current_state) {  // Handle UI updates
+      case cereal::SpeechToText::State::BEGIN:
+        if (!hideTimer->isActive()) animateOverlay(true);
+        updateText("Hello, I'm listening");
+        hideTimer->start(30000);
+        break;
+      case cereal::SpeechToText::State::EMPTY:
+        updateText("Sorry, I didn't catch that");
+        hideTimer->start(8000);
+        break;
+      case cereal::SpeechToText::State::ERROR:
+        updateText("Sorry, an error occorred");
+        hideTimer->start(8000);
+        break;
+      case cereal::SpeechToText::State::FINAL:
+      case cereal::SpeechToText::State::NONE:
+        updateText(QString::fromStdString(sm["speechToText"].getSpeechToText().getTranscript()));
+        hideTimer->start(request_state == cereal::SpeechToText::State::FINAL ? 8000 : 30000);
+        break;
+    }
+  }
+}

selfdrive/ui/qt/widgets/assistant.h

@@ -0,0 +1,25 @@
+#pragma once
+
+#include "selfdrive/ui/ui.h"
+#include <QLabel>
+#include <QPropertyAnimation>
+
+class AssistantOverlay : public QLabel {
+  Q_OBJECT
+
+public:
+  explicit AssistantOverlay(QWidget *parent = nullptr);
+  void animateOverlay(bool show);
+
+private:
+  void updateText(const QString text);
+  void startHideTimer();
+  QTimer *hideTimer;
+  QPropertyAnimation *showAnimation;
+  QPropertyAnimation *hideAnimation;
+  int finalWidth;
+
+private slots:
+  void updateState(const UIState &s);
+
+};

selfdrive/ui/qt/window.cc

@@ -8,10 +8,12 @@ MainWindow::MainWindow(QWidget *parent) : QWidget(parent) {
   main_layout = new QStackedLayout(this);
   main_layout->setMargin(0);
 
+  assistantOverlay = new AssistantOverlay(this);
   homeWindow = new HomeWindow(this);
   main_layout->addWidget(homeWindow);
   QObject::connect(homeWindow, &HomeWindow::openSettings, this, &MainWindow::openSettings);
   QObject::connect(homeWindow, &HomeWindow::closeSettings, this, &MainWindow::closeSettings);
+  QObject::connect(homeWindow, &HomeWindow::requestRaiseAssistantOverlay, this, &MainWindow::raiseAssistantOverlay);
 
   settingsWindow = new SettingsWindow(this);
   main_layout->addWidget(settingsWindow);
@@ -37,11 +39,13 @@ MainWindow::MainWindow(QWidget *parent) : QWidget(parent) {
     if (!offroad) {
       closeSettings();
     }
+    assistantOverlay->raise();
   });
   QObject::connect(device(), &Device::interactiveTimeout, [=]() {
     if (main_layout->currentWidget() == settingsWindow) {
       closeSettings();
     }
+    assistantOverlay->raise();
   });
 
   // load fonts
@@ -68,6 +72,7 @@ MainWindow::MainWindow(QWidget *parent) : QWidget(parent) {
 void MainWindow::openSettings(int index, const QString &param) {
   main_layout->setCurrentWidget(settingsWindow);
   settingsWindow->setCurrentPanel(index, param);
+  assistantOverlay->raise();
 }
 
 void MainWindow::closeSettings() {
@@ -80,6 +85,7 @@ void MainWindow::closeSettings() {
       homeWindow->showMapPanel(true);
     }
   }
+  assistantOverlay->raise();
 }
 
 bool MainWindow::eventFilter(QObject *obj, QEvent *event) {

selfdrive/ui/qt/window.h

@@ -6,6 +6,7 @@
 #include "selfdrive/ui/qt/home.h"
 #include "selfdrive/ui/qt/offroad/onboarding.h"
 #include "selfdrive/ui/qt/offroad/settings.h"
+#include "selfdrive/ui/qt/widgets/assistant.h"
 
 class MainWindow : public QWidget {
   Q_OBJECT
@@ -17,9 +18,11 @@ class MainWindow : public QWidget {
   bool eventFilter(QObject *obj, QEvent *event) override;
   void openSettings(int index = 0, const QString &param = "");
   void closeSettings();
+  void raiseAssistantOverlay() {assistantOverlay->raise();}
 
   QStackedLayout *main_layout;
   HomeWindow *homeWindow;
   SettingsWindow *settingsWindow;
   OnboardingWindow *onboardingWindow;
+  AssistantOverlay *assistantOverlay;
 };

selfdrive/ui/ui.cc

@@ -248,7 +248,7 @@ UIState::UIState(QObject *parent) : QObject(parent) {
   sm = std::make_unique<SubMaster, const std::initializer_list<const char *>>({
     "modelV2", "controlsState", "liveCalibration", "radarState", "deviceState",
     "pandaStates", "carParams", "driverMonitoringState", "carState", "liveLocationKalman", "driverStateV2",
-    "wideRoadCameraState", "managerState", "navInstruction", "navRoute", "uiPlan",
+    "wideRoadCameraState", "managerState", "navInstruction", "navRoute", "uiPlan", "speechToText",
   });
 
   Params params;

system/assistant/nav_setter.py

@@ -0,0 +1,59 @@
+from cereal import messaging, log
+from openpilot.common.params import Params
+import json
+
+STTState = log.SpeechToText.State
+
+sm = messaging.SubMaster(["speechToText"])
+import os
+import re
+import requests
+import urllib.parse
+
+def get_coordinates_from_transcript(transcript, proximity, mapbox_access_token):
+  # Regular expression to find 'navigate to' or 'directions to' followed by an address
+  pattern = r'\b(navigate to|directions to)\b\s+(.*?)(\.|$)'
+  # Search for the pattern in the transcript
+  match = re.search(pattern, transcript, re.IGNORECASE)
+  if match:
+    address = match.group(2).strip()
+    encoded_address = urllib.parse.quote(address)
+    mapbox_url = f"https://api.mapbox.com/geocoding/v5/mapbox.places/{encoded_address}.json?access_token=pk.eyJ1IjoicnlsZXltY2MiLCJhIjoiY2xjeDl5aGp4MTBmeDNzb2Vua2QyNWN1bSJ9.CrbD-j1LQkBdOqyWcZneyQ"
+    response = requests.get(mapbox_url)
+    if response.status_code == 200:
+      data = response.json()
+      # Assuming the first result is the most relevant
+      if data['features']:
+        coordinates = {
+          "latitude": data['features'][0]['geometry']['coordinates'][0],
+          "longitude": data['features'][0]['geometry']['coordinates'][1],
+        }
+        return coordinates
+      print("No coordinates")
+    print(f"Mapbox API error: {response.status_code}")
+  return False
+
+def main():
+  params = Params()
+  mapbox_access_token = os.environ["MAPBOX_TOKEN"]
+  while True:
+    dest = False
+    transcript: str = ""
+    sm.update(0)
+    if sm.updated["speechToText"]:
+      transcript = sm["speechToText"].transcript
+      if not sm["speechToText"].state == log.SpeechToText.State.final:
+        print(f'Interim result: {transcript}')
+      else:
+        print(f'Final result: {transcript}')
+        proximity = params.get("LastGPSPosition")
+        print(proximity)
+        dest = get_coordinates_from_transcript(transcript,proximity, mapbox_access_token)
+        if dest:
+          params.put("NavDestination", json.dumps(dest))
+          print(dest)
+          dest = False
+
+if __name__ == "__main__":
+    main()
+