demo.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Demo the Text-to-Speech module"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Available ONNX Runtime providers:\n",
      "TensorrtExecutionProvider\n",
      "CUDAExecutionProvider\n",
      "AzureExecutionProvider\n",
      "CPUExecutionProvider\n"
     ]
    }
   ],
   "source": [
    "import glados.tts as tts\n",
    "import sounddevice as sd\n",
    "\n",
    "\n",
    "import onnxruntime as ort\n",
    "\n",
    "print(\"Available ONNX Runtime providers:\")\n",
    "for provider in ort.get_available_providers():\n",
    "    print(provider)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Loading libespeak-ng.so.1\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[0;93m2024-05-01 08:05:27.820180399 [W:onnxruntime:, transformer_memcpy.cc:74 ApplyImpl] 28 Memcpy nodes are added to the graph torch_jit for CUDAExecutionProvider. It might have negative impact on performance (including unable to run CUDA graph). Set session_options.log_severity_level=1 to see the detail logs before this message.\u001b[m\n",
      "\u001b[0;93m2024-05-01 08:05:27.828684869 [W:onnxruntime:, session_state.cc:1166 VerifyEachNodeIsAssignedToAnEp] Some nodes were not assigned to the preferred execution providers which may or may not have an negative impact on performance. e.g. ORT explicitly assigns shape related ops to CPU to improve perf.\u001b[m\n",
      "\u001b[0;93m2024-05-01 08:05:27.828690294 [W:onnxruntime:, session_state.cc:1168 VerifyEachNodeIsAssignedToAnEp] Rerunning with verbose output on a non-minimal build will show node assignments.\u001b[m\n"
     ]
    }
   ],
   "source": [
    "# Instantiate the TTS engine\n",
    "glados_tts = tts.TTSEngine()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Generate the audio.\n",
    "# Glados is spelt incorrectly on purpose to make the pronunciation more accurate.\n",
    "audio = glados_tts.generate_speech_audio(\"Hello, my name is Gladohs. I am an AI created by Aperture Science.\")\n",
    "\n",
    "# Play the audio\n",
    "sd.play(audio, tts.RATE)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Demo the Automatic speech recogntion system\n",
    "This will detect and transcribe your voice. In this demo, it will then get GlaDOS to repeat back to you what was heard."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "Could not load whisper.",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[5], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mglados\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mvoice_recognition\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mvr\u001b[39;00m\n",
      "File \u001b[0;32m~/GlaDOS/glados/voice_recognition.py:9\u001b[0m\n\u001b[1;32m      6\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mLevenshtein\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m distance\n\u001b[1;32m      7\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mloguru\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m logger\n\u001b[0;32m----> 9\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m asr, vad\n\u001b[1;32m     11\u001b[0m ASR_MODEL_PATH \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m./models/ggml-medium-32-2.en.bin\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m     12\u001b[0m VAD_MODEL_PATH \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m./models/silero_vad.onnx\u001b[39m\u001b[38;5;124m\"\u001b[39m\n",
      "File \u001b[0;32m~/GlaDOS/glados/asr.py:5\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mctypes\u001b[39;00m\n\u001b[1;32m      3\u001b[0m \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mnumpy\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m \u001b[38;5;21;01mnp\u001b[39;00m\n\u001b[0;32m----> 5\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01m.\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m whisper_cpp_wrapper\n\u001b[1;32m      7\u001b[0m LANG \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124men\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m      8\u001b[0m WORD_LEVEL_TIMINGS \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m\n",
      "File \u001b[0;32m~/GlaDOS/glados/whisper_cpp_wrapper.py:861\u001b[0m\n\u001b[1;32m    858\u001b[0m add_library_search_dirs([])\n\u001b[1;32m    860\u001b[0m \u001b[38;5;66;03m# Begin libraries\u001b[39;00m\n\u001b[0;32m--> 861\u001b[0m _libs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mwhisper\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[43mload_library\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mwhisper\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m    863\u001b[0m \u001b[38;5;66;03m# 1 libraries\u001b[39;00m\n\u001b[1;32m    864\u001b[0m \u001b[38;5;66;03m# End libraries\u001b[39;00m\n\u001b[1;32m    865\u001b[0m \n\u001b[1;32m    866\u001b[0m \u001b[38;5;66;03m# No modules\u001b[39;00m\n\u001b[1;32m    868\u001b[0m __uint32_t \u001b[38;5;241m=\u001b[39m c_uint  \u001b[38;5;66;03m# /usr/include/x86_64-linux-gnu/bits/types.h: 42\u001b[39;00m\n",
      "File \u001b[0;32m~/GlaDOS/glados/whisper_cpp_wrapper.py:547\u001b[0m, in \u001b[0;36mLibraryLoader.__call__\u001b[0;34m(self, libname)\u001b[0m\n\u001b[1;32m    544\u001b[0m     \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m:  \u001b[38;5;66;03m# pylint: disable=broad-except\u001b[39;00m\n\u001b[1;32m    545\u001b[0m         \u001b[38;5;28;01mpass\u001b[39;00m\n\u001b[0;32m--> 547\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mImportError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCould not load \u001b[39m\u001b[38;5;132;01m%s\u001b[39;00m\u001b[38;5;124m.\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m libname)\n",
      "\u001b[0;31mImportError\u001b[0m: Could not load whisper."
     ]
    }
   ],
   "source": [
    "import glados.voice_recognition as vr"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def say_text(text: str):\n",
    "    \"\"\"Say text using text-to-speech engine\n",
    "    \"\"\"\n",
    "    audio = glados_tts.generate_speech_audio(text)\n",
    "    sd.play(audio, tts.RATE)\n",
    "    sd.wait()\n",
    "\n",
    "# Instantiate VoiceRecognition class with the say_text function\n",
    "demo = vr.VoiceRecognition(function=say_text)\n",
    "\n",
    "# Start the demo\n",
    "demo.start()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}