Merge pull request #64 from Bontempogianpaolo1/master

Integration with microphone and webRTC module

examples/microphone_and_webRTC_integration/README.md

@@ -0,0 +1,28 @@
+
+In this example, an integration with the microphone and the webRTC VAD has been done.  I used [this](https://github.com/mozilla/DeepSpeech-examples/tree/r0.8/mic_vad_streaming) as a draft.
+Here a short video to present the results:
+
+https://user-images.githubusercontent.com/28188499/116685087-182ff100-a9b2-11eb-927d-ed9f621226ee.mp4
+
+# Requirements:
+The libraries used for the following example are:
+```
+Python == 3.6.9
+webrtcvad >= 2.0.10
+torchaudio >= 0.8.1
+torch >= 1.8.1
+halo >= 0.0.31
+Soundfile >= 0.13.3
+```
+Using pip3:
+```
+pip3 install webrtcvad
+pip3 install torchaudio
+pip3 install torch
+pip3 install halo
+pip3 install soundfile
+```
+Moreover, to make the code easier, the default sample_rate is 16KHz without resampling.
+
+This example has been tested on ``` ubuntu 18.04.3 LTS```
+

examples/microphone_and_webRTC_integration/microphone_and_webRTC_integration.py

@@ -0,0 +1,201 @@
+import collections, queue
+import numpy as np
+import pyaudio
+import webrtcvad
+from halo import Halo
+import torch
+import torchaudio
+
+class Audio(object):
+    """Streams raw audio from microphone. Data is received in a separate thread, and stored in a buffer, to be read from."""
+
+    FORMAT = pyaudio.paInt16
+    # Network/VAD rate-space
+    RATE_PROCESS = 16000
+    CHANNELS = 1
+    BLOCKS_PER_SECOND = 50
+
+    def __init__(self, callback=None, device=None, input_rate=RATE_PROCESS):
+        def proxy_callback(in_data, frame_count, time_info, status):
+            #pylint: disable=unused-argument
+            callback(in_data)
+            return (None, pyaudio.paContinue)
+        if callback is None: callback = lambda in_data: self.buffer_queue.put(in_data)
+        self.buffer_queue = queue.Queue()
+        self.device = device
+        self.input_rate = input_rate
+        self.sample_rate = self.RATE_PROCESS
+        self.block_size = int(self.RATE_PROCESS / float(self.BLOCKS_PER_SECOND))
+        self.block_size_input = int(self.input_rate / float(self.BLOCKS_PER_SECOND))
+        self.pa = pyaudio.PyAudio()
+
+        kwargs = {
+            'format': self.FORMAT,
+            'channels': self.CHANNELS,
+            'rate': self.input_rate,
+            'input': True,
+            'frames_per_buffer': self.block_size_input,
+            'stream_callback': proxy_callback,
+        }
+
+        self.chunk = None
+        # if not default device
+        if self.device:
+            kwargs['input_device_index'] = self.device
+
+        self.stream = self.pa.open(**kwargs)
+        self.stream.start_stream()
+
+    def read(self):
+        """Return a block of audio data, blocking if necessary."""
+        return self.buffer_queue.get()
+
+    def destroy(self):
+        self.stream.stop_stream()
+        self.stream.close()
+        self.pa.terminate()
+
+    frame_duration_ms = property(lambda self: 1000 * self.block_size // self.sample_rate)
+
+
+class VADAudio(Audio):
+    """Filter & segment audio with voice activity detection."""
+
+    def __init__(self, aggressiveness=3, device=None, input_rate=None):
+        super().__init__(device=device, input_rate=input_rate)
+        self.vad = webrtcvad.Vad(aggressiveness)
+
+    def frame_generator(self):
+        """Generator that yields all audio frames from microphone."""
+        if self.input_rate == self.RATE_PROCESS:
+            while True:
+                yield self.read()
+        else:
+            raise Exception("Resampling required")
+
+    def vad_collector(self, padding_ms=300, ratio=0.75, frames=None):
+        """Generator that yields series of consecutive audio frames comprising each utterence, separated by yielding a single None.
+            Determines voice activity by ratio of frames in padding_ms. Uses a buffer to include padding_ms prior to being triggered.
+            Example: (frame, ..., frame, None, frame, ..., frame, None, ...)
+                      |---utterence---|        |---utterence---|
+        """
+        if frames is None: frames = self.frame_generator()
+        num_padding_frames = padding_ms // self.frame_duration_ms
+        ring_buffer = collections.deque(maxlen=num_padding_frames)
+        triggered = False
+
+        for frame in frames:
+            if len(frame) < 640:
+                return
+
+            is_speech = self.vad.is_speech(frame, self.sample_rate)
+
+            if not triggered:
+                ring_buffer.append((frame, is_speech))
+                num_voiced = len([f for f, speech in ring_buffer if speech])
+                if num_voiced > ratio * ring_buffer.maxlen:
+                    triggered = True
+                    for f, s in ring_buffer:
+                        yield f
+                    ring_buffer.clear()
+
+            else:
+                yield frame
+                ring_buffer.append((frame, is_speech))
+                num_unvoiced = len([f for f, speech in ring_buffer if not speech])
+                if num_unvoiced > ratio * ring_buffer.maxlen:
+                    triggered = False
+                    yield None
+                    ring_buffer.clear()
+
+def main(ARGS):
+    # Start audio with VAD
+    vad_audio = VADAudio(aggressiveness=ARGS.webRTC_aggressiveness,
+                         device=ARGS.device,
+                         input_rate=ARGS.rate)
+
+    print("Listening (ctrl-C to exit)...")
+    frames = vad_audio.vad_collector()
+
+    # load silero VAD
+    torchaudio.set_audio_backend("soundfile")
+    model, utils = torch.hub.load(repo_or_dir='snakers4/silero-vad',
+                                    model=ARGS.silaro_model_name,
+                                    force_reload= ARGS.reload)
+    (get_speech_ts,_,_, _,_, _, _) = utils
+
+
+    # Stream from microphone to DeepSpeech using VAD
+    spinner = None
+    if not ARGS.nospinner:
+        spinner = Halo(spinner='line')
+    wav_data = bytearray()
+    for frame in frames:
+        if frame is not None:
+            if spinner: spinner.start()
+
+            wav_data.extend(frame)
+        else:
+            if spinner: spinner.stop()
+            print("webRTC has detected a possible speech")
+
+            newsound= np.frombuffer(wav_data,np.int16)
+            audio_float32=Int2Float(newsound)
+            time_stamps =get_speech_ts(audio_float32, model,num_steps=ARGS.num_steps,trig_sum=ARGS.trig_sum,neg_trig_sum=ARGS.neg_trig_sum,
+                                    num_samples_per_window=ARGS.num_samples_per_window,min_speech_samples=ARGS.min_speech_samples,
+                                    min_silence_samples=ARGS.min_silence_samples)
+
+            if(len(time_stamps)>0):
+                print("silero VAD has detected a possible speech")
+            else:
+                print("silero VAD has detected a noise")
+            print()
+            wav_data = bytearray()
+
+
+def Int2Float(sound):
+    _sound = np.copy(sound)  #
+    abs_max = np.abs(_sound).max()
+    _sound = _sound.astype('float32')
+    if abs_max > 0:
+        _sound *= 1/abs_max
+    audio_float32 = torch.from_numpy(_sound.squeeze())
+    return audio_float32
+
+if __name__ == '__main__':
+    DEFAULT_SAMPLE_RATE = 16000
+
+    import argparse
+    parser = argparse.ArgumentParser(description="Stream from microphone to webRTC and silero VAD")
+
+    parser.add_argument('-v', '--webRTC_aggressiveness', type=int, default=3,
+                        help="Set aggressiveness of webRTC: an integer between 0 and 3, 0 being the least aggressive about filtering out non-speech, 3 the most aggressive. Default: 3")
+    parser.add_argument('--nospinner', action='store_true',
+                        help="Disable spinner")
+    parser.add_argument('-d', '--device', type=int, default=None,
+                        help="Device input index (Int) as listed by pyaudio.PyAudio.get_device_info_by_index(). If not provided, falls back to PyAudio.get_default_device().")
+
+    parser.add_argument('-name', '--silaro_model_name', type=str, default="silero_vad",
+                        help="select the name of the model. You can select between 'silero_vad',''silero_vad_micro','silero_vad_micro_8k','silero_vad_mini','silero_vad_mini_8k'")
+    parser.add_argument('--reload', action='store_true',help="download the last version of the silero vad")
+
+    parser.add_argument('-ts', '--trig_sum', type=float, default=0.25,
+                        help="overlapping windows are used for each audio chunk, trig sum defines average probability among those windows for switching into triggered state (speech state)")
+
+    parser.add_argument('-nts', '--neg_trig_sum', type=float, default=0.07,
+                        help="same as trig_sum, but for switching from triggered to non-triggered state (non-speech)")
+
+    parser.add_argument('-N', '--num_steps', type=int, default=8,
+                        help="nubmer of overlapping windows to split audio chunk into (we recommend 4 or 8)")
+
+    parser.add_argument('-nspw', '--num_samples_per_window', type=int, default=4000,
+                        help="number of samples in each window, our models were trained using 4000 samples (250 ms) per window, so this is preferable value (lesser values reduce quality)")
+
+    parser.add_argument('-msps', '--min_speech_samples', type=int, default=10000,
+                        help="minimum speech chunk duration in samples")
+
+    parser.add_argument('-msis', '--min_silence_samples', type=int, default=500,
+                        help=" minimum silence duration in samples between to separate speech chunks")
+    ARGS = parser.parse_args()
+    ARGS.rate=DEFAULT_SAMPLE_RATE
+    main(ARGS)