v6: Tauri (Rust + TS) rewrite, Python as ML sidecar

Cross-platform Stream Studio rewrite. UI/ffmpeg/exports move to Rust+TypeScript; Python shrinks to a thin sidecar that runs inaSpeechSegmenter and emits line-delimited JSON. Detection slice extracted from stream-studio.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

v6_stream_studio_tauri/detect.rs

@@ -0,0 +1,222 @@
+//! Singing-segment detection.
+//!
+//! Rust owns the fast/IO parts (audio extraction via ffmpeg, progress events,
+//! and all file exports). The Python sidecar owns *only* the ML step
+//! (inaSpeechSegmenter + TensorFlow), which has no native equivalent.
+//!
+//! Protocol: the sidecar prints one JSON object per line to stdout:
+//!   {"type":"progress","message":"…"}
+//!   {"type":"result","segments":[{index,start,end,duration}, …],
+//!                     "stats":{"music":123.4,"speech":456.7,…}}
+//!   {"type":"error","message":"…"}
+
+use std::io::{BufRead, BufReader};
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+
+use serde::{Deserialize, Serialize};
+use serde_json::Value;
+use tauri::{AppHandle, Emitter};
+
+use crate::tools::hide_window;
+use crate::AppState;
+
+#[derive(Clone, Debug, Serialize, Deserialize)]
+pub struct Segment {
+    pub index: u32,
+    pub start: f64,
+    pub end: f64,
+    pub duration: f64,
+}
+
+#[derive(Clone, Serialize)]
+pub struct DetectResult {
+    pub segments: Vec<Segment>,
+    pub stats: Value,
+}
+
+#[derive(Clone, Serialize)]
+struct DetectProgress {
+    message: String,
+}
+
+fn emit(app: &AppHandle, message: impl Into<String>) {
+    let _ = app.emit("detect-progress", DetectProgress { message: message.into() });
+}
+
+/// Locate the sidecar script: bundled resources first, dev tree as fallback.
+fn sidecar_script() -> Option<PathBuf> {
+    if let Ok(p) = std::env::var("STREAM_STUDIO_SIDECAR") {
+        let pb = PathBuf::from(p);
+        if pb.exists() {
+            return Some(pb);
+        }
+    }
+    if let Ok(exe) = std::env::current_exe() {
+        if let Some(base) = exe.parent() {
+            for sub in ["sidecar", "resources/sidecar", "../sidecar"] {
+                let cand = base.join(sub).join("detect.py");
+                if cand.exists() {
+                    return Some(cand);
+                }
+            }
+        }
+    }
+    // Dev fallback: <crate>/../sidecar/detect.py
+    let dev = Path::new(env!("CARGO_MANIFEST_DIR")).join("../sidecar/detect.py");
+    if dev.exists() {
+        return Some(dev);
+    }
+    None
+}
+
+/// Build the python invocation. Prefers the Windows `py -3.10` launcher
+/// (TensorFlow does not yet support 3.12+), then a venv, then plain `python`.
+fn python_command(script: &Path) -> Command {
+    if let Ok(custom) = std::env::var("STREAM_STUDIO_PYTHON") {
+        let mut c = Command::new(custom);
+        c.arg(script);
+        return c;
+    }
+    #[cfg(windows)]
+    {
+        // The py launcher lets us pin 3.10 regardless of the default install.
+        let mut c = Command::new("py");
+        c.args(["-3.10"]).arg(script);
+        return c;
+    }
+    #[allow(unreachable_code)]
+    {
+        let mut c = Command::new("python");
+        c.arg(script);
+        c
+    }
+}
+
+/// Extract 16 kHz mono WAV (the format inaSpeechSegmenter expects).
+fn extract_audio(app: &AppHandle, ffmpeg: &str, video: &str, wav: &Path) -> Result<(), String> {
+    emit(app, "Extracting audio (16 kHz mono)…");
+    let mut cmd = Command::new(ffmpeg);
+    cmd.args([
+        "-y", "-i", video,
+        "-vn", "-acodec", "pcm_s16le", "-ar", "16000", "-ac", "1",
+        "-hide_banner", "-loglevel", "error",
+        &wav.to_string_lossy(),
+    ]);
+    cmd.stdout(Stdio::null()).stderr(Stdio::piped());
+    hide_window(&mut cmd);
+    let out = cmd.output().map_err(|e| format!("ffmpeg audio extract failed: {e}"))?;
+    if !out.status.success() {
+        return Err(format!("Audio extraction failed:\n{}", String::from_utf8_lossy(&out.stderr)));
+    }
+    Ok(())
+}
+
+pub fn run(
+    app: &AppHandle,
+    state: &AppState,
+    ffmpeg: &str,
+    video: String,
+    gap: f64,
+    min_duration: f64,
+    padding: f64,
+) -> Result<DetectResult, String> {
+    state.cancel.store(false, std::sync::atomic::Ordering::SeqCst);
+
+    let script = sidecar_script().ok_or(
+        "Detection sidecar not found. Expected sidecar/detect.py next to the app.",
+    )?;
+
+    // Temp WAV beside the source video.
+    let wav = Path::new(&video)
+        .with_extension("")
+        .to_string_lossy()
+        .to_string();
+    let wav = PathBuf::from(format!("{wav}.stream_studio.wav"));
+
+    extract_audio(app, ffmpeg, &video, &wav)?;
+
+    let mut cmd = python_command(&script);
+    cmd.args([
+        "--wav", &wav.to_string_lossy(),
+        "--gap", &gap.to_string(),
+        "--min-duration", &min_duration.to_string(),
+        "--padding", &padding.to_string(),
+    ]);
+    cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
+    cmd.env("PYTHONUNBUFFERED", "1").env("PYTHONIOENCODING", "utf-8");
+    hide_window(&mut cmd);
+
+    emit(app, "Loading detection model…");
+    let mut child = cmd.spawn().map_err(|e| {
+        format!("Could not start Python sidecar ({e}). Is Python 3.10 + inaSpeechSegmenter installed?")
+    })?;
+    let stdout = child.stdout.take().ok_or("No sidecar stdout")?;
+    let stderr = child.stderr.take();
+    *state.proc.lock().unwrap() = Some(child);
+
+    let mut result: Option<DetectResult> = None;
+    let mut err_msg: Option<String> = None;
+
+    for line in BufReader::new(stdout).lines() {
+        let line = match line {
+            Ok(l) => l,
+            Err(_) => break,
+        };
+        let line = line.trim();
+        if line.is_empty() {
+            continue;
+        }
+        let parsed: Value = match serde_json::from_str(line) {
+            Ok(v) => v,
+            Err(_) => continue, // ignore stray non-JSON output
+        };
+        match parsed.get("type").and_then(|t| t.as_str()) {
+            Some("progress") => {
+                if let Some(m) = parsed.get("message").and_then(|m| m.as_str()) {
+                    emit(app, m);
+                }
+            }
+            Some("result") => {
+                let segments: Vec<Segment> = serde_json::from_value(
+                    parsed.get("segments").cloned().unwrap_or(Value::Array(vec![])),
+                )
+                .unwrap_or_default();
+                let stats = parsed.get("stats").cloned().unwrap_or(Value::Null);
+                result = Some(DetectResult { segments, stats });
+            }
+            Some("error") => {
+                err_msg = parsed
+                    .get("message")
+                    .and_then(|m| m.as_str())
+                    .map(|s| s.to_string());
+            }
+            _ => {}
+        }
+    }
+
+    let mut child = state.proc.lock().unwrap().take().ok_or("Sidecar vanished")?;
+    let status = child.wait().map_err(|e| format!("sidecar wait failed: {e}"))?;
+    let mut stderr_text = String::new();
+    if let Some(mut se) = stderr {
+        use std::io::Read;
+        let _ = se.read_to_string(&mut stderr_text);
+    }
+
+    let _ = std::fs::remove_file(&wav);
+
+    if state.cancel.load(std::sync::atomic::Ordering::SeqCst) {
+        return Err("__cancelled__".into());
+    }
+    if let Some(m) = err_msg {
+        return Err(m);
+    }
+    if let Some(r) = result {
+        return Ok(r);
+    }
+    if !status.success() {
+        let tail: String = stderr_text.chars().rev().take(800).collect::<String>().chars().rev().collect();
+        return Err(format!("Detection failed.\n{tail}"));
+    }
+    Err("Detection produced no result.".into())
+}

v6_stream_studio_tauri/detect.ts

@@ -0,0 +1,129 @@
+// Detect view — find singing segments in one video, export editor markers.
+
+import { h, clear, basename, fmtDur } from "../dom";
+import { makeProgress, toast, renderSongs } from "../ui";
+import {
+  pickVideo, detectSongs, cancel, exportSegments, onDetectProgress, reveal,
+  type Segment,
+} from "../api";
+
+export interface View { el: HTMLElement; onDrop: (paths: string[]) => void; setVideo: (p: string) => void; }
+
+const FPS = ["29.97", "25", "23.976", "30", "60"];
+
+function numField(label: string, value: number, step = "1"): [HTMLElement, HTMLInputElement] {
+  const input = h("input", { class: "input", type: "number", value: String(value), step, min: "0" }) as HTMLInputElement;
+  return [h("div", { class: "field" }, h("label", {}, label), input), input];
+}
+
+export function buildDetect(): View {
+  let running = false;
+  let segments: Segment[] = [];
+  let videoPath = "";
+
+  const videoInput = h("input", { class: "input grow", placeholder: "Select a video…", readonly: true }) as HTMLInputElement;
+  const [gapF, gapI] = numField("Merge gap (s)", 10);
+  const [minF, minI] = numField("Min duration (s)", 20);
+  const [padF, padI] = numField("Padding (s)", 2);
+  const fpsSel = h("select", { class: "input" }, ...FPS.map((f) => h("option", { value: f }, f))) as HTMLSelectElement;
+
+  const detectBtn = h("button", { class: "btn primary lg", disabled: true }, "🎤  Detect Songs") as HTMLButtonElement;
+  const stopBtn = h("button", { class: "btn danger", disabled: true }, "■ Stop") as HTMLButtonElement;
+  const exportBtn = h("button", { class: "btn", disabled: true }, "⬇ Export EDL / CSV") as HTMLButtonElement;
+  const revealBtn = h("button", { class: "btn ghost", style: "display:none" }, "Show in folder") as HTMLButtonElement;
+  const progress = makeProgress();
+  const statsEl = h("div", { class: "pills" });
+  const songsEl = h("div", { class: "songs" });
+  let lastExportDir = "";
+
+  function setVideo(p: string) {
+    videoPath = p;
+    videoInput.value = basename(p);
+    videoInput.title = p;
+    detectBtn.disabled = running || !p;
+  }
+
+  async function detect() {
+    if (!videoPath) return;
+    running = true;
+    detectBtn.disabled = true;
+    stopBtn.disabled = false;
+    exportBtn.disabled = true;
+    revealBtn.style.display = "none";
+    clear(songsEl); clear(statsEl);
+    progress.reset();
+    progress.indeterminate(true, "Starting detection…");
+
+    const un = await onDetectProgress((msg) => {
+      progress.indeterminate(true, msg);
+      progress.appendLog(msg);
+    });
+
+    try {
+      const res = await detectSongs(videoPath, +gapI.value, +minI.value, +padI.value);
+      segments = res.segments;
+      progress.indeterminate(false);
+      progress.set(1, `Found ${segments.length} singing segment(s).`);
+      renderSongs(songsEl, segments);
+      const totalSing = segments.reduce((a, s) => a + s.duration, 0);
+      statsEl.replaceChildren(
+        h("span", { class: "pill" }, `${segments.length} songs`),
+        h("span", { class: "pill" }, `singing ${fmtDur(totalSing)}`),
+        ...Object.entries(res.stats || {}).map(([k, v]) =>
+          h("span", { class: "pill" }, `${k} ${fmtDur(v as number)}`)),
+      );
+      exportBtn.disabled = segments.length === 0;
+      if (!segments.length) toast("No singing detected — try a larger gap or smaller min duration.", "");
+    } catch (e) {
+      const msg = String(e);
+      progress.indeterminate(false);
+      if (msg.includes("__cancelled__")) progress.set(0, "Stopped by user.");
+      else { progress.set(0, "Detection failed."); toast(msg, "err"); }
+    } finally {
+      un();
+      running = false;
+      stopBtn.disabled = true;
+      detectBtn.disabled = !videoPath;
+    }
+  }
+
+  exportBtn.addEventListener("click", async () => {
+    if (!segments.length) return;
+    try {
+      const res = await exportSegments(segments, videoPath, null, +fpsSel.value);
+      lastExportDir = res.files[0] ?? "";
+      toast(`Exported ${res.files.length} files`, "ok");
+      if (lastExportDir) revealBtn.style.display = "";
+    } catch (e) { toast(String(e), "err"); }
+  });
+  revealBtn.addEventListener("click", () => lastExportDir && reveal(lastExportDir));
+  detectBtn.addEventListener("click", detect);
+  stopBtn.addEventListener("click", () => cancel());
+
+  const el = h("div", { class: "view" },
+    h("div", { class: "card" },
+      h("h2", {}, h("span", { class: "step" }, "1"), "Source Video"),
+      h("div", { class: "row" }, videoInput,
+        h("button", { class: "btn", onclick: async () => { const p = await pickVideo(); if (p) setVideo(p); } }, "Browse…")),
+      h("div", { class: "hint", style: "margin-top:8px" }, "Tip: drop a file here, or use a Concat output."),
+    ),
+    h("div", { class: "card" },
+      h("h2", {}, h("span", { class: "step" }, "2"), "Detection Settings"),
+      h("div", { class: "row wrap" },
+        h("div", { class: "grow", style: "min-width:130px" }, gapF),
+        h("div", { class: "grow", style: "min-width:130px" }, minF),
+        h("div", { class: "grow", style: "min-width:130px" }, padF),
+        h("div", { class: "field", style: "min-width:120px" }, h("label", {}, "FPS (export)"), fpsSel)),
+      h("div", { class: "hint", style: "margin-top:8px" }, "Singing = \"music\". More chatting between songs → raise the gap. Short covers → lower the min duration."),
+    ),
+    h("div", { class: "card" },
+      h("h2", {}, h("span", { class: "step" }, "3"), "Run"),
+      h("div", { class: "row", style: "margin-bottom:14px" }, detectBtn, stopBtn, exportBtn, revealBtn),
+      progress.root,
+      h("div", { style: "margin-top:12px" }, statsEl),
+      h("div", { style: "margin-top:10px" }, songsEl),
+    ),
+  );
+
+  return { el, onDrop: (paths) => { if (paths[0]) setVideo(paths[0]); }, setVideo };
+}

v6_stream_studio_tauri/sidecar_detect.py

@@ -0,0 +1,114 @@
+#!/usr/bin/env python3
+"""
+Stream Studio — singing-detection sidecar.
+
+This is the ONLY Python in the project. It exists solely because the ML model
+(inaSpeechSegmenter, which classifies singing voice as "music") is TensorFlow-
+based and has no native Rust/JS equivalent. Everything else — UI, ffmpeg
+concat, audio extraction, file exports — is handled by the Tauri/Rust host.
+
+Contract (line-delimited JSON on stdout):
+    {"type": "progress", "message": "..."}
+    {"type": "result", "segments": [{"index","start","end","duration"}...],
+                        "stats": {"music": 123.4, "speech": 456.7, ...}}
+    {"type": "error",   "message": "..."}
+
+The host extracts a 16 kHz mono WAV and passes it via --wav, so this script
+never touches ffmpeg.
+
+Run with Python 3.10 (TensorFlow does not support 3.12+):
+    py -3.10 detect.py --wav audio.wav --gap 10 --min-duration 20 --padding 2
+"""
+
+import argparse
+import json
+import sys
+import time
+
+
+def emit(obj: dict) -> None:
+    """Write one JSON record per line and flush immediately."""
+    sys.stdout.write(json.dumps(obj, ensure_ascii=False) + "\n")
+    sys.stdout.flush()
+
+
+def progress(msg: str) -> None:
+    emit({"type": "progress", "message": msg})
+
+
+def merge_singing_segments(segments, max_gap, min_duration, padding):
+    """Filter for 'music' segments and merge nearby ones into songs."""
+    music = sorted((s, e) for label, s, e in segments if label == "music")
+    if not music:
+        return []
+
+    merged = []
+    cur_start, cur_end = music[0]
+    for s, e in music[1:]:
+        if s - cur_end <= max_gap:
+            cur_end = max(cur_end, e)
+        else:
+            merged.append((cur_start, cur_end))
+            cur_start, cur_end = s, e
+    merged.append((cur_start, cur_end))
+
+    results = []
+    idx = 1
+    for s, e in merged:
+        if (e - s) >= min_duration:
+            ps = max(0.0, s - padding)
+            pe = e + padding
+            results.append({
+                "index": idx,
+                "start": round(ps, 3),
+                "end": round(pe, 3),
+                "duration": round(pe - ps, 3),
+            })
+            idx += 1
+    return results
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--wav", required=True)
+    ap.add_argument("--gap", type=float, default=10.0)
+    ap.add_argument("--min-duration", type=float, default=20.0)
+    ap.add_argument("--padding", type=float, default=2.0)
+    args = ap.parse_args()
+
+    try:
+        progress("Loading detection model (first run downloads weights)…")
+        from inaSpeechSegmenter import Segmenter
+    except ImportError:
+        emit({
+            "type": "error",
+            "message": (
+                "inaSpeechSegmenter is not installed for this Python.\n"
+                "Install it with:\n"
+                "  py -3.10 -m pip install inaSpeechSegmenter tensorflow"
+            ),
+        })
+        return 1
+
+    try:
+        t0 = time.time()
+        seg = Segmenter(vad_engine="smn", detect_gender=False)
+        progress("Analysing audio — this can take a while on long streams…")
+        raw = seg(args.wav)
+        progress(f"Segmentation done in {time.time() - t0:.0f}s ({len(raw)} raw segments).")
+
+        stats = {}
+        for label, s, e in raw:
+            stats[label] = round(stats.get(label, 0.0) + (e - s), 1)
+
+        songs = merge_singing_segments(raw, args.gap, args.min_duration, args.padding)
+        progress(f"Found {len(songs)} singing segment(s) after merge + filter.")
+        emit({"type": "result", "segments": songs, "stats": stats})
+        return 0
+    except Exception as exc:  # noqa: BLE001 — surface any model/runtime error to the host
+        emit({"type": "error", "message": f"{type(exc).__name__}: {exc}"})
+        return 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())