aime.rolandi/voice_recognition
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

whisper.cpp en tiempo real con tiny

Browse Source
This commit is contained in:
aime-rolandi
2025-06-19 15:50:54 -03:00
parent 9908d84b8c
commit 77f87c3655
5 changed files with 227 additions and 93 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
recognition_VAD/lib/recognition_VAD/application.ex
View File

@ -12,6 +12,8 @@ defmodule Recognition_VAD.Application do
{DNSCluster, query: Application.get_env(:recognition_VAD, :dns_cluster_query) || :ignore},
{Phoenix.PubSub, name: Recognition_VAD.PubSub},
Recognition_VAD.AudioProcessor,
Recognition_VAD.WhisperStreamer,
# Start the Finch HTTP client for sending emails
{Finch, name: Recognition_VAD.Finch},
# Start a worker by calling: Recognition_VAD.Worker.start_link(arg)

46
recognition_VAD/lib/recognition_VAD/whisper.ex Normal file
View File

@ -0,0 +1,46 @@
defmodule Recognition_VAD.Whisper do
@default_model "ggml-tiny.bin"
@script_path "/home/aime-pc2/i_m/whisper.cpp/transcribe.sh"
require Logger
def transcribe(path, model \\ @default_model) do
path_to_run = convert_path_to_wsl(path)
args = [@script_path, path_to_run, model]
case System.cmd("wsl", args, stderr_to_stdout: true) do
{output, 0} ->
text = extract_transcription(output)
Logger.info("📝 Transcripción: #{text}")
{:ok, text}
{error_output, _} ->
Logger.error("❌ Error al transcribir con whisper: #{error_output}")
{:error, error_output}
end
end
defp convert_path_to_wsl(path) do
if String.starts_with?(path, "C:/") do
path
|> String.replace_prefix("C:/", "/mnt/c/")
|> String.replace("\\", "/")
else
path
end
end
defp extract_transcription(output) do
output
|> String.split("\n")
|> Enum.filter(fn line ->
line =~ ~r/[\p{L}\p{N}]/u and
not String.starts_with?(line, "whisper_") and
not String.starts_with?(line, "system_info") and
not String.starts_with?(line, "main: ") and
not String.starts_with?(line, "whisper_print_timings:")
end)
|> Enum.join(" ")
|> String.trim()
end
end

56
recognition_VAD/lib/recognition_VAD/whisper_streamer.ex Normal file
View File

@ -0,0 +1,56 @@
defmodule Recognition_VAD.WhisperStreamer do
use GenServer
require Logger
@transcribe_interval 2000 # cada 2 segundos
@max_chunks 100 # máximo a mantener en memoria
def start_link(_opts) do
GenServer.start_link(__MODULE__, %{chunks: [], sample_rate: 48000}, name: __MODULE__)
end
def push_chunk(chunk, sample_rate) do
GenServer.cast(__MODULE__, {:chunk, chunk, sample_rate})
end
@impl true
def init(state) do
schedule_transcription()
{:ok, state}
end
@impl true
def handle_cast({:chunk, binary, sr}, %{chunks: chunks} = state) do
new_chunks = [binary | chunks] |> Enum.take(@max_chunks)
{:noreply, %{state | chunks: new_chunks, sample_rate: sr}}
end
@impl true
def handle_info(:transcribe_timer, %{chunks: []} = state) do
# Si no hay audio, solo reprogramamos
schedule_transcription()
{:noreply, state}
end
def handle_info(:transcribe_timer, %{chunks: chunks, sample_rate: sr} = state) do
Task.start(fn ->
path = "C:/Users/rolan/i_m/voice_recognition/recognition_VAD/tmp/realtime_#{System.system_time(:millisecond)}.wav"
:ok = Recognition_VAD.WavWriter.write_pcm_chunks_to_wav(Enum.reverse(chunks), sr, path)
case Recognition_VAD.Whisper.transcribe(path) do
{:ok, text} when byte_size(text) > 0 ->
Phoenix.PubSub.broadcast(Recognition_VAD.PubSub, "audio_output", {:realtime, %{"text" => text}})
_ ->
Logger.debug("⏱ Nada para transcribir o error")
end
end)
schedule_transcription()
{:noreply, %{state | chunks: []}}
end
defp schedule_transcription do
Process.send_after(self(), :transcribe_timer, @transcribe_interval)
end
end

11
recognition_VAD/lib/recognition_VAD_web/channels/data_channel.ex
View File

@ -6,8 +6,15 @@ defmodule Recognition_VADWeb.DataChannel do
{:ok, socket}
end
# Parcial
def handle_info({:realtime, msg}, socket) do
push(socket, "realtime", msg)
{:noreply, socket}
end
# Completo
def handle_info({:broadcast_audio, msg}, socket) do
push(socket, "transcription", Jason.decode!(msg))
push(socket, "transcription", msg)
{:noreply, socket}
end
@ -15,7 +22,7 @@ defmodule Recognition_VADWeb.DataChannel do
def handle_in("audio_chunk", %{"data" => base64_chunk, "sample_rate" => sample_rate}, socket) do
case Base.decode64(base64_chunk) do
{:ok, binary_audio} ->
GenServer.cast(Recognition_VAD.AudioProcessor, {:chunk, binary_audio, sample_rate})
Recognition_VAD.WhisperStreamer.push_chunk(binary_audio, sample_rate)
{:noreply, socket}
:error ->

205
recognition_VAD/lib/recognition_VAD_web/live/stt/test_with_channel.ex
View File

@ -7,114 +7,137 @@ defmodule Recognition_VADWeb.Stt.TestWithChannel do
def render(assigns) do
~H"""
<div id="container">
<div id="status">Presioná "Start Recording"…</div>
<button id="startButton">Start Recording</button>
<button id="stopButton" disabled>Stop Recording</button>
<div id="container">
<div id="status">Presioná "Start Recording"…</div>
<button id="startButton">Start Recording</button>
<button id="stopButton" disabled>Stop Recording</button>
<div id="transcriptionContainer">
<div id="transcription" class="realtime"></div>
</div>
<div id="transcriptionContainer">
<div id="transcription" class="realtime"></div>
</div>
<div id="fullTextContainer">
<div id="fullText"></div>
</div>
<script type="module">
import { Socket } from "https://cdn.skypack.dev/phoenix";
<script type="module">
import { Socket } from "https://cdn.skypack.dev/phoenix";
const statusDiv = document.getElementById("status");
const transcriptionDiv = document.getElementById("transcription");
const startButton = document.getElementById("startButton");
const stopButton = document.getElementById("stopButton");
const statusDiv = document.getElementById("status");
const transcriptionDiv = document.getElementById("transcription");
const fullTextDiv = document.getElementById("fullText");
const startButton = document.getElementById("startButton");
const stopButton = document.getElementById("stopButton");
let socket, channel;
let audioContext, mediaStream, mediaProcessor;
let socket, channel;
let audioContext, mediaStream, mediaProcessor;
async function startRecording() {
startButton.disabled = true;
stopButton.disabled = false;
statusDiv.textContent = "Recording…";
transcriptionDiv.textContent = "";
fullTextDiv.textContent = "";
socket = new Socket("ws://localhost:4000/socket");
socket.connect();
channel = socket.channel("data:lobby");
channel.join()
.receive("ok", () => {
statusDiv.textContent = "🎙 Conectado a Phoenix STT";
console.log("Canal conectado");
})
.receive("error", () => {
statusDiv.textContent = "❌ Error al conectar";
console.error("Error al conectar canal");
});
channel.on("realtime", payload => {
const words = payload.text.split(" ");
const lastWord = words.pop();
transcriptionDiv.innerHTML = `${words.join(" ")} <span class="last-word">${lastWord}</span>`;
});
channel.on("fullSentence", payload => {
fullTextDiv.innerHTML += payload.text + " ";
async function startRecording() {
startButton.disabled = true;
stopButton.disabled = false;
statusDiv.textContent = "🎙 Grabando…";
transcriptionDiv.innerHTML = "";
});
audioContext = new AudioContext();
mediaStream = await navigator.mediaDevices.getUserMedia({ audio: true });
const input = audioContext.createMediaStreamSource(mediaStream);
socket = new Socket("ws://localhost:4000/socket");
socket.connect();
mediaProcessor = audioContext.createScriptProcessor(1024, 1, 1);
mediaProcessor.onaudioprocess = (event) => {
const float32Array = event.inputBuffer.getChannelData(0);
const int16Array = new Int16Array(float32Array.length);
for (let i = 0; i < float32Array.length; i++) {
int16Array[i] = Math.max(-1, Math.min(1, float32Array[i])) * 0x7FFF;
}
channel = socket.channel("data:lobby");
const base64Audio = btoa(String.fromCharCode(...new Uint8Array(int16Array.buffer)));
channel.push("audio_chunk", {
data: base64Audio,
sample_rate: audioContext.sampleRate
channel.join()
.receive("ok", () => {
statusDiv.textContent = "✅ Conectado a Phoenix STT";
console.log("Canal conectado");
})
.receive("error", () => {
statusDiv.textContent = "❌ Error al conectar canal";
console.error("Error al conectar canal");
});
// Realtime parcial (palabras mientras habla)
let partialTranscript = "";
channel.on("realtime", payload => {
const words = payload.text.split(" ");
const lastWord = words.pop();
const rest = words.join(" ");
if (rest.length > 0) {
partialTranscript += rest + " ";
}
transcriptionDiv.innerHTML = `
${partialTranscript}<span class="last-word">${lastWord}</span>
`;
});
};
input.connect(mediaProcessor);
mediaProcessor.connect(audioContext.destination);
}
function stopRecording() {
stopButton.disabled = true;
startButton.disabled = false;
statusDiv.textContent = "🛑 Grabación detenida.";
// Frase completa (después de procesar chunks)
channel.on("transcription", payload => {
const sentence = payload.text.trim();
if (sentence.length > 0) {
partialTranscript = ""; // reseteamos el parcial
const span = document.createElement("div");
span.className = "sentence";
span.textContent = sentence;
transcriptionDiv.appendChild(span);
transcriptionDiv.innerHTML += "<br />";
}
});
// ✅ Enviamos evento especial para guardar
if (channel) {
channel.push("save_audio", {});
// Audio setup
audioContext = new AudioContext();
mediaStream = await navigator.mediaDevices.getUserMedia({ audio: true });
const input = audioContext.createMediaStreamSource(mediaStream);
mediaProcessor = audioContext.createScriptProcessor(1024, 1, 1);
mediaProcessor.onaudioprocess = (event) => {
const float32Array = event.inputBuffer.getChannelData(0);
const int16Array = new Int16Array(float32Array.length);
for (let i = 0; i < float32Array.length; i++) {
const s = Math.max(-1, Math.min(1, float32Array[i]));
int16Array[i] = s < 0 ? s * 0x8000 : s * 0x7FFF;
}
const base64Audio = btoa(String.fromCharCode(...new Uint8Array(int16Array.buffer)));
channel.push("audio_chunk", {
data: base64Audio,
sample_rate: audioContext.sampleRate
});
};
input.connect(mediaProcessor);
mediaProcessor.connect(audioContext.destination);
}
if (mediaProcessor) mediaProcessor.disconnect();
if (audioContext) audioContext.close();
if (mediaStream) mediaStream.getTracks().forEach(track => track.stop());
if (channel) channel.leave();
if (socket) socket.disconnect();
}
function stopRecording() {
stopButton.disabled = true;
startButton.disabled = false;
statusDiv.textContent = "🛑 Grabación detenida.";
document.getElementById("startButton").onclick = startRecording;
document.getElementById("stopButton").onclick = stopRecording;
</script>
if (mediaProcessor) mediaProcessor.disconnect();
if (audioContext) audioContext.close();
if (mediaStream) mediaStream.getTracks().forEach(track => track.stop());
if (channel) channel.leave();
if (socket) socket.disconnect();
}
<style>
.last-word {
font-weight: bold;
color: orange;
}
</style>
</div>
document.getElementById("startButton").onclick = startRecording;
document.getElementById("stopButton").onclick = stopRecording;
</script>
<style>
.last-word {
font-weight: bold;
color: orange;
}
#transcriptionContainer {
margin-top: 1rem;
font-family: sans-serif;
font-size: 1.1rem;
}
.sentence {
margin-bottom: 0.5rem;
}
</style>
</div>
"""
end
end