From e44341d781b45d098d350c9d462ebb607f3a1063 Mon Sep 17 00:00:00 2001
From: Freddy Boulton <alfonsoboulton@gmail.com>
Date: Wed, 26 Feb 2025 12:37:12 -0500
Subject: [PATCH] Moonshine live (#86)

* add code

* moonshine

---------

Co-authored-by: Freddy Boulton <freddyboulton@hf-freddy.local>
---
 demo/moonshine_live/README.md           |  16 ++++++
 demo/moonshine_live/app.py              |  64 ++++++++++++++++++++++++
 demo/moonshine_live/default-favicon.ico | Bin 0 -> 6518 bytes
 demo/moonshine_live/requirements.txt    |   3 ++
 4 files changed, 83 insertions(+)
 create mode 100644 demo/moonshine_live/README.md
 create mode 100644 demo/moonshine_live/app.py
 create mode 100644 demo/moonshine_live/default-favicon.ico
 create mode 100644 demo/moonshine_live/requirements.txt

diff --git a/demo/moonshine_live/README.md b/demo/moonshine_live/README.md
new file mode 100644
index 0000000..d541a3d
--- /dev/null
+++ b/demo/moonshine_live/README.md
@@ -0,0 +1,16 @@
+---
+title: Moonshine Live Transcription
+emoji: 🌕
+colorFrom: purple
+colorTo: red
+sdk: gradio
+sdk_version: 5.17.0
+app_file: app.py
+pinned: false
+license: mit
+short_description: Real-time captions with Moonshine ONNX
+tags: [webrtc, websocket, gradio, secret|TWILIO_ACCOUNT_SID, secret|TWILIO_ACCOUNT_SID, secret|TWILIO_AUTH_TOKEN]
+models: [onnx-community/moonshine-base-ONNX, UsefulSensors/moonshine-base]
+---
+
+Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
\ No newline at end of file
diff --git a/demo/moonshine_live/app.py b/demo/moonshine_live/app.py
new file mode 100644
index 0000000..bbb05cb
--- /dev/null
+++ b/demo/moonshine_live/app.py
@@ -0,0 +1,64 @@
+from fastrtc import (
+    Stream,
+    AdditionalOutputs,
+    audio_to_float32,
+    ReplyOnPause,
+    get_twilio_turn_credentials,
+)
+from functools import lru_cache
+import gradio as gr
+from typing import Generator, Literal
+from numpy.typing import NDArray
+import numpy as np
+from moonshine_onnx import MoonshineOnnxModel, load_tokenizer
+
+
+@lru_cache(maxsize=None)
+def load_moonshine(
+    model_name: Literal["moonshine/base", "moonshine/tiny"],
+) -> MoonshineOnnxModel:
+    return MoonshineOnnxModel(model_name=model_name)
+
+
+tokenizer = load_tokenizer()
+
+
+def stt(
+    audio: tuple[int, NDArray[np.int16 | np.float32]],
+    model_name: Literal["moonshine/base", "moonshine/tiny"],
+) -> Generator[AdditionalOutputs, None, None]:
+    moonshine = load_moonshine(model_name)
+    sr, audio_np = audio  # type: ignore
+    if audio_np.dtype == np.int16:
+        audio_np = audio_to_float32(audio)
+    if audio_np.ndim == 1:
+        audio_np = audio_np.reshape(1, -1)
+    tokens = moonshine.generate(audio_np)
+    yield AdditionalOutputs(tokenizer.decode_batch(tokens)[0])
+
+
+stream = Stream(
+    ReplyOnPause(stt, input_sample_rate=16000),
+    modality="audio",
+    mode="send",
+    ui_args={
+        "title": "Live Captions by Moonshine",
+        "icon": "default-favicon.ico",
+        "icon_button_color": "#5c5c5c",
+        "pulse_color": "#a7c6fc",
+        "icon_radius": 0,
+    },
+    rtc_configuration=get_twilio_turn_credentials(),
+    additional_inputs=[
+        gr.Radio(
+            choices=["moonshine/base", "moonshine/tiny"],
+            value="moonshine/base",
+            label="Model",
+        )
+    ],
+    additional_outputs=[gr.Textbox(label="Captions")],
+    additional_outputs_handler=lambda prev, current: (prev + "\n" + current).strip(),
+)
+
+if __name__ == "__main__":
+    stream.ui.launch()
diff --git a/demo/moonshine_live/default-favicon.ico b/demo/moonshine_live/default-favicon.ico
new file mode 100644
index 0000000000000000000000000000000000000000..0a7c372dbf95aad931396cd060949c135c05efa1
GIT binary patch
literal 6518
zcmeHLIc_3B6s@sDPGE8(%NaTANJxYX0gDhqB!CD&V}uZ!IKw#Q6yuxNC-4D?9Fvq^
zd0gl-ddB|Motc(O^?AIdRdiSL+n68b$|REpbK2Y{jJYw!q|@)8uZ^h!y9EXc82@lg
z#sL4y(P%VM^?LoT*=#-+i^b<$E_YWb6jJB3!{IOiehs<@=nt5!8`*5uj{|%j;8Sy4
z@6~FxGT`51?7c?LLhpSneD=Wa^Z9&6a#_vv5@Q;PNiM4;m(@%oF{Y82<g!|FS<N&O
zV;YG`E~_P%)l4HXrolB~`~AM1PN#M}9$Q=+wp1z|e}ATToL;YICzFX?EEaaTT%HVv
z4F&^StybOjbM$Jr+b^Th$j;~U6aN2t`FS>**>1OM%jNRRcDv22R;xc4d&xOhJFe3>
zt-lX;yIo?l+57_j7uo~SAKZ72<G5bOX7Ez$_4+sFM>pQ<v-dCa`3{)dC({@|G2DLo
zc3=;U<G?kk-bbV6hVH{rbEEF#A^CoNF+Z_=L}GH!U^VuHCye0doXlc1%lrTfdQP%T
ziG7fD+y~kBn&+&{Vm0&EV9Zm4X|fnslf}5qQ-d*24W`LrTum0^GEWW0JT;gmi*Yqs
zjLSSV81vL%nk>fEWHB!D)L_h0gK4rDSChrK%u|CgPYtHYVq8rY<1$YT#ymBcCW~=3
zS)QxuV_2rdyys_gXdfiASp8XUr_-^aeN?4VIr~4voe#^G>r48jFZXj!8&{th^*rM2
zQ=^_s)IK+AK6=-XK3Vm<lXDE}vx$%DJfimLalMBK>BFQZ&i5G7H>AdzUfzR*EZjwN
LS%J$6oUFhr>0(2^

literal 0
HcmV?d00001

diff --git a/demo/moonshine_live/requirements.txt b/demo/moonshine_live/requirements.txt
new file mode 100644
index 0000000..acde84b
--- /dev/null
+++ b/demo/moonshine_live/requirements.txt
@@ -0,0 +1,3 @@
+fastrtc[vad]
+useful-moonshine-onnx@git+https://git@github.com/usefulsensors/moonshine.git#subdirectory=moonshine-onnx
+twilio
\ No newline at end of file