diff --git a/v6_stream_studio_tauri/detect.rs b/v6_stream_studio_tauri/detect.rs new file mode 100644 index 0000000..342ece4 --- /dev/null +++ b/v6_stream_studio_tauri/detect.rs @@ -0,0 +1,222 @@ +//! Singing-segment detection. +//! +//! Rust owns the fast/IO parts (audio extraction via ffmpeg, progress events, +//! and all file exports). The Python sidecar owns *only* the ML step +//! (inaSpeechSegmenter + TensorFlow), which has no native equivalent. +//! +//! Protocol: the sidecar prints one JSON object per line to stdout: +//! {"type":"progress","message":"…"} +//! {"type":"result","segments":[{index,start,end,duration}, …], +//! "stats":{"music":123.4,"speech":456.7,…}} +//! {"type":"error","message":"…"} + +use std::io::{BufRead, BufReader}; +use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; + +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use tauri::{AppHandle, Emitter}; + +use crate::tools::hide_window; +use crate::AppState; + +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct Segment { + pub index: u32, + pub start: f64, + pub end: f64, + pub duration: f64, +} + +#[derive(Clone, Serialize)] +pub struct DetectResult { + pub segments: Vec, + pub stats: Value, +} + +#[derive(Clone, Serialize)] +struct DetectProgress { + message: String, +} + +fn emit(app: &AppHandle, message: impl Into) { + let _ = app.emit("detect-progress", DetectProgress { message: message.into() }); +} + +/// Locate the sidecar script: bundled resources first, dev tree as fallback. +fn sidecar_script() -> Option { + if let Ok(p) = std::env::var("STREAM_STUDIO_SIDECAR") { + let pb = PathBuf::from(p); + if pb.exists() { + return Some(pb); + } + } + if let Ok(exe) = std::env::current_exe() { + if let Some(base) = exe.parent() { + for sub in ["sidecar", "resources/sidecar", "../sidecar"] { + let cand = base.join(sub).join("detect.py"); + if cand.exists() { + return Some(cand); + } + } + } + } + // Dev fallback: /../sidecar/detect.py + let dev = Path::new(env!("CARGO_MANIFEST_DIR")).join("../sidecar/detect.py"); + if dev.exists() { + return Some(dev); + } + None +} + +/// Build the python invocation. Prefers the Windows `py -3.10` launcher +/// (TensorFlow does not yet support 3.12+), then a venv, then plain `python`. +fn python_command(script: &Path) -> Command { + if let Ok(custom) = std::env::var("STREAM_STUDIO_PYTHON") { + let mut c = Command::new(custom); + c.arg(script); + return c; + } + #[cfg(windows)] + { + // The py launcher lets us pin 3.10 regardless of the default install. + let mut c = Command::new("py"); + c.args(["-3.10"]).arg(script); + return c; + } + #[allow(unreachable_code)] + { + let mut c = Command::new("python"); + c.arg(script); + c + } +} + +/// Extract 16 kHz mono WAV (the format inaSpeechSegmenter expects). +fn extract_audio(app: &AppHandle, ffmpeg: &str, video: &str, wav: &Path) -> Result<(), String> { + emit(app, "Extracting audio (16 kHz mono)…"); + let mut cmd = Command::new(ffmpeg); + cmd.args([ + "-y", "-i", video, + "-vn", "-acodec", "pcm_s16le", "-ar", "16000", "-ac", "1", + "-hide_banner", "-loglevel", "error", + &wav.to_string_lossy(), + ]); + cmd.stdout(Stdio::null()).stderr(Stdio::piped()); + hide_window(&mut cmd); + let out = cmd.output().map_err(|e| format!("ffmpeg audio extract failed: {e}"))?; + if !out.status.success() { + return Err(format!("Audio extraction failed:\n{}", String::from_utf8_lossy(&out.stderr))); + } + Ok(()) +} + +pub fn run( + app: &AppHandle, + state: &AppState, + ffmpeg: &str, + video: String, + gap: f64, + min_duration: f64, + padding: f64, +) -> Result { + state.cancel.store(false, std::sync::atomic::Ordering::SeqCst); + + let script = sidecar_script().ok_or( + "Detection sidecar not found. Expected sidecar/detect.py next to the app.", + )?; + + // Temp WAV beside the source video. + let wav = Path::new(&video) + .with_extension("") + .to_string_lossy() + .to_string(); + let wav = PathBuf::from(format!("{wav}.stream_studio.wav")); + + extract_audio(app, ffmpeg, &video, &wav)?; + + let mut cmd = python_command(&script); + cmd.args([ + "--wav", &wav.to_string_lossy(), + "--gap", &gap.to_string(), + "--min-duration", &min_duration.to_string(), + "--padding", &padding.to_string(), + ]); + cmd.stdout(Stdio::piped()).stderr(Stdio::piped()); + cmd.env("PYTHONUNBUFFERED", "1").env("PYTHONIOENCODING", "utf-8"); + hide_window(&mut cmd); + + emit(app, "Loading detection model…"); + let mut child = cmd.spawn().map_err(|e| { + format!("Could not start Python sidecar ({e}). Is Python 3.10 + inaSpeechSegmenter installed?") + })?; + let stdout = child.stdout.take().ok_or("No sidecar stdout")?; + let stderr = child.stderr.take(); + *state.proc.lock().unwrap() = Some(child); + + let mut result: Option = None; + let mut err_msg: Option = None; + + for line in BufReader::new(stdout).lines() { + let line = match line { + Ok(l) => l, + Err(_) => break, + }; + let line = line.trim(); + if line.is_empty() { + continue; + } + let parsed: Value = match serde_json::from_str(line) { + Ok(v) => v, + Err(_) => continue, // ignore stray non-JSON output + }; + match parsed.get("type").and_then(|t| t.as_str()) { + Some("progress") => { + if let Some(m) = parsed.get("message").and_then(|m| m.as_str()) { + emit(app, m); + } + } + Some("result") => { + let segments: Vec = serde_json::from_value( + parsed.get("segments").cloned().unwrap_or(Value::Array(vec![])), + ) + .unwrap_or_default(); + let stats = parsed.get("stats").cloned().unwrap_or(Value::Null); + result = Some(DetectResult { segments, stats }); + } + Some("error") => { + err_msg = parsed + .get("message") + .and_then(|m| m.as_str()) + .map(|s| s.to_string()); + } + _ => {} + } + } + + let mut child = state.proc.lock().unwrap().take().ok_or("Sidecar vanished")?; + let status = child.wait().map_err(|e| format!("sidecar wait failed: {e}"))?; + let mut stderr_text = String::new(); + if let Some(mut se) = stderr { + use std::io::Read; + let _ = se.read_to_string(&mut stderr_text); + } + + let _ = std::fs::remove_file(&wav); + + if state.cancel.load(std::sync::atomic::Ordering::SeqCst) { + return Err("__cancelled__".into()); + } + if let Some(m) = err_msg { + return Err(m); + } + if let Some(r) = result { + return Ok(r); + } + if !status.success() { + let tail: String = stderr_text.chars().rev().take(800).collect::().chars().rev().collect(); + return Err(format!("Detection failed.\n{tail}")); + } + Err("Detection produced no result.".into()) +} diff --git a/v6_stream_studio_tauri/detect.ts b/v6_stream_studio_tauri/detect.ts new file mode 100644 index 0000000..9d8be28 --- /dev/null +++ b/v6_stream_studio_tauri/detect.ts @@ -0,0 +1,129 @@ +// Detect view — find singing segments in one video, export editor markers. + +import { h, clear, basename, fmtDur } from "../dom"; +import { makeProgress, toast, renderSongs } from "../ui"; +import { + pickVideo, detectSongs, cancel, exportSegments, onDetectProgress, reveal, + type Segment, +} from "../api"; + +export interface View { el: HTMLElement; onDrop: (paths: string[]) => void; setVideo: (p: string) => void; } + +const FPS = ["29.97", "25", "23.976", "30", "60"]; + +function numField(label: string, value: number, step = "1"): [HTMLElement, HTMLInputElement] { + const input = h("input", { class: "input", type: "number", value: String(value), step, min: "0" }) as HTMLInputElement; + return [h("div", { class: "field" }, h("label", {}, label), input), input]; +} + +export function buildDetect(): View { + let running = false; + let segments: Segment[] = []; + let videoPath = ""; + + const videoInput = h("input", { class: "input grow", placeholder: "Select a video…", readonly: true }) as HTMLInputElement; + const [gapF, gapI] = numField("Merge gap (s)", 10); + const [minF, minI] = numField("Min duration (s)", 20); + const [padF, padI] = numField("Padding (s)", 2); + const fpsSel = h("select", { class: "input" }, ...FPS.map((f) => h("option", { value: f }, f))) as HTMLSelectElement; + + const detectBtn = h("button", { class: "btn primary lg", disabled: true }, "🎤 Detect Songs") as HTMLButtonElement; + const stopBtn = h("button", { class: "btn danger", disabled: true }, "■ Stop") as HTMLButtonElement; + const exportBtn = h("button", { class: "btn", disabled: true }, "⬇ Export EDL / CSV") as HTMLButtonElement; + const revealBtn = h("button", { class: "btn ghost", style: "display:none" }, "Show in folder") as HTMLButtonElement; + const progress = makeProgress(); + const statsEl = h("div", { class: "pills" }); + const songsEl = h("div", { class: "songs" }); + let lastExportDir = ""; + + function setVideo(p: string) { + videoPath = p; + videoInput.value = basename(p); + videoInput.title = p; + detectBtn.disabled = running || !p; + } + + async function detect() { + if (!videoPath) return; + running = true; + detectBtn.disabled = true; + stopBtn.disabled = false; + exportBtn.disabled = true; + revealBtn.style.display = "none"; + clear(songsEl); clear(statsEl); + progress.reset(); + progress.indeterminate(true, "Starting detection…"); + + const un = await onDetectProgress((msg) => { + progress.indeterminate(true, msg); + progress.appendLog(msg); + }); + + try { + const res = await detectSongs(videoPath, +gapI.value, +minI.value, +padI.value); + segments = res.segments; + progress.indeterminate(false); + progress.set(1, `Found ${segments.length} singing segment(s).`); + renderSongs(songsEl, segments); + const totalSing = segments.reduce((a, s) => a + s.duration, 0); + statsEl.replaceChildren( + h("span", { class: "pill" }, `${segments.length} songs`), + h("span", { class: "pill" }, `singing ${fmtDur(totalSing)}`), + ...Object.entries(res.stats || {}).map(([k, v]) => + h("span", { class: "pill" }, `${k} ${fmtDur(v as number)}`)), + ); + exportBtn.disabled = segments.length === 0; + if (!segments.length) toast("No singing detected — try a larger gap or smaller min duration.", ""); + } catch (e) { + const msg = String(e); + progress.indeterminate(false); + if (msg.includes("__cancelled__")) progress.set(0, "Stopped by user."); + else { progress.set(0, "Detection failed."); toast(msg, "err"); } + } finally { + un(); + running = false; + stopBtn.disabled = true; + detectBtn.disabled = !videoPath; + } + } + + exportBtn.addEventListener("click", async () => { + if (!segments.length) return; + try { + const res = await exportSegments(segments, videoPath, null, +fpsSel.value); + lastExportDir = res.files[0] ?? ""; + toast(`Exported ${res.files.length} files`, "ok"); + if (lastExportDir) revealBtn.style.display = ""; + } catch (e) { toast(String(e), "err"); } + }); + revealBtn.addEventListener("click", () => lastExportDir && reveal(lastExportDir)); + detectBtn.addEventListener("click", detect); + stopBtn.addEventListener("click", () => cancel()); + + const el = h("div", { class: "view" }, + h("div", { class: "card" }, + h("h2", {}, h("span", { class: "step" }, "1"), "Source Video"), + h("div", { class: "row" }, videoInput, + h("button", { class: "btn", onclick: async () => { const p = await pickVideo(); if (p) setVideo(p); } }, "Browse…")), + h("div", { class: "hint", style: "margin-top:8px" }, "Tip: drop a file here, or use a Concat output."), + ), + h("div", { class: "card" }, + h("h2", {}, h("span", { class: "step" }, "2"), "Detection Settings"), + h("div", { class: "row wrap" }, + h("div", { class: "grow", style: "min-width:130px" }, gapF), + h("div", { class: "grow", style: "min-width:130px" }, minF), + h("div", { class: "grow", style: "min-width:130px" }, padF), + h("div", { class: "field", style: "min-width:120px" }, h("label", {}, "FPS (export)"), fpsSel)), + h("div", { class: "hint", style: "margin-top:8px" }, "Singing = \"music\". More chatting between songs → raise the gap. Short covers → lower the min duration."), + ), + h("div", { class: "card" }, + h("h2", {}, h("span", { class: "step" }, "3"), "Run"), + h("div", { class: "row", style: "margin-bottom:14px" }, detectBtn, stopBtn, exportBtn, revealBtn), + progress.root, + h("div", { style: "margin-top:12px" }, statsEl), + h("div", { style: "margin-top:10px" }, songsEl), + ), + ); + + return { el, onDrop: (paths) => { if (paths[0]) setVideo(paths[0]); }, setVideo }; +} diff --git a/v6_stream_studio_tauri/sidecar_detect.py b/v6_stream_studio_tauri/sidecar_detect.py new file mode 100644 index 0000000..c4f93c9 --- /dev/null +++ b/v6_stream_studio_tauri/sidecar_detect.py @@ -0,0 +1,114 @@ +#!/usr/bin/env python3 +""" +Stream Studio — singing-detection sidecar. + +This is the ONLY Python in the project. It exists solely because the ML model +(inaSpeechSegmenter, which classifies singing voice as "music") is TensorFlow- +based and has no native Rust/JS equivalent. Everything else — UI, ffmpeg +concat, audio extraction, file exports — is handled by the Tauri/Rust host. + +Contract (line-delimited JSON on stdout): + {"type": "progress", "message": "..."} + {"type": "result", "segments": [{"index","start","end","duration"}...], + "stats": {"music": 123.4, "speech": 456.7, ...}} + {"type": "error", "message": "..."} + +The host extracts a 16 kHz mono WAV and passes it via --wav, so this script +never touches ffmpeg. + +Run with Python 3.10 (TensorFlow does not support 3.12+): + py -3.10 detect.py --wav audio.wav --gap 10 --min-duration 20 --padding 2 +""" + +import argparse +import json +import sys +import time + + +def emit(obj: dict) -> None: + """Write one JSON record per line and flush immediately.""" + sys.stdout.write(json.dumps(obj, ensure_ascii=False) + "\n") + sys.stdout.flush() + + +def progress(msg: str) -> None: + emit({"type": "progress", "message": msg}) + + +def merge_singing_segments(segments, max_gap, min_duration, padding): + """Filter for 'music' segments and merge nearby ones into songs.""" + music = sorted((s, e) for label, s, e in segments if label == "music") + if not music: + return [] + + merged = [] + cur_start, cur_end = music[0] + for s, e in music[1:]: + if s - cur_end <= max_gap: + cur_end = max(cur_end, e) + else: + merged.append((cur_start, cur_end)) + cur_start, cur_end = s, e + merged.append((cur_start, cur_end)) + + results = [] + idx = 1 + for s, e in merged: + if (e - s) >= min_duration: + ps = max(0.0, s - padding) + pe = e + padding + results.append({ + "index": idx, + "start": round(ps, 3), + "end": round(pe, 3), + "duration": round(pe - ps, 3), + }) + idx += 1 + return results + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--wav", required=True) + ap.add_argument("--gap", type=float, default=10.0) + ap.add_argument("--min-duration", type=float, default=20.0) + ap.add_argument("--padding", type=float, default=2.0) + args = ap.parse_args() + + try: + progress("Loading detection model (first run downloads weights)…") + from inaSpeechSegmenter import Segmenter + except ImportError: + emit({ + "type": "error", + "message": ( + "inaSpeechSegmenter is not installed for this Python.\n" + "Install it with:\n" + " py -3.10 -m pip install inaSpeechSegmenter tensorflow" + ), + }) + return 1 + + try: + t0 = time.time() + seg = Segmenter(vad_engine="smn", detect_gender=False) + progress("Analysing audio — this can take a while on long streams…") + raw = seg(args.wav) + progress(f"Segmentation done in {time.time() - t0:.0f}s ({len(raw)} raw segments).") + + stats = {} + for label, s, e in raw: + stats[label] = round(stats.get(label, 0.0) + (e - s), 1) + + songs = merge_singing_segments(raw, args.gap, args.min_duration, args.padding) + progress(f"Found {len(songs)} singing segment(s) after merge + filter.") + emit({"type": "result", "segments": songs, "stats": stats}) + return 0 + except Exception as exc: # noqa: BLE001 — surface any model/runtime error to the host + emit({"type": "error", "message": f"{type(exc).__name__}: {exc}"}) + return 1 + + +if __name__ == "__main__": + sys.exit(main())