Compare commits
4 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 33c275a114 | |||
| b9573e8644 | |||
| 98a4d37162 | |||
| 91df8d3cec |
222
detect.rs
Normal file
222
detect.rs
Normal file
@@ -0,0 +1,222 @@
|
|||||||
|
//! Singing-segment detection.
|
||||||
|
//!
|
||||||
|
//! Rust owns the fast/IO parts (audio extraction via ffmpeg, progress events,
|
||||||
|
//! and all file exports). The Python sidecar owns *only* the ML step
|
||||||
|
//! (inaSpeechSegmenter + TensorFlow), which has no native equivalent.
|
||||||
|
//!
|
||||||
|
//! Protocol: the sidecar prints one JSON object per line to stdout:
|
||||||
|
//! {"type":"progress","message":"…"}
|
||||||
|
//! {"type":"result","segments":[{index,start,end,duration}, …],
|
||||||
|
//! "stats":{"music":123.4,"speech":456.7,…}}
|
||||||
|
//! {"type":"error","message":"…"}
|
||||||
|
|
||||||
|
use std::io::{BufRead, BufReader};
|
||||||
|
use std::path::{Path, PathBuf};
|
||||||
|
use std::process::{Command, Stdio};
|
||||||
|
|
||||||
|
use serde::{Deserialize, Serialize};
|
||||||
|
use serde_json::Value;
|
||||||
|
use tauri::{AppHandle, Emitter};
|
||||||
|
|
||||||
|
use crate::tools::hide_window;
|
||||||
|
use crate::AppState;
|
||||||
|
|
||||||
|
#[derive(Clone, Debug, Serialize, Deserialize)]
|
||||||
|
pub struct Segment {
|
||||||
|
pub index: u32,
|
||||||
|
pub start: f64,
|
||||||
|
pub end: f64,
|
||||||
|
pub duration: f64,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
pub struct DetectResult {
|
||||||
|
pub segments: Vec<Segment>,
|
||||||
|
pub stats: Value,
|
||||||
|
}
|
||||||
|
|
||||||
|
#[derive(Clone, Serialize)]
|
||||||
|
struct DetectProgress {
|
||||||
|
message: String,
|
||||||
|
}
|
||||||
|
|
||||||
|
fn emit(app: &AppHandle, message: impl Into<String>) {
|
||||||
|
let _ = app.emit("detect-progress", DetectProgress { message: message.into() });
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Locate the sidecar script: bundled resources first, dev tree as fallback.
|
||||||
|
fn sidecar_script() -> Option<PathBuf> {
|
||||||
|
if let Ok(p) = std::env::var("STREAM_STUDIO_SIDECAR") {
|
||||||
|
let pb = PathBuf::from(p);
|
||||||
|
if pb.exists() {
|
||||||
|
return Some(pb);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if let Ok(exe) = std::env::current_exe() {
|
||||||
|
if let Some(base) = exe.parent() {
|
||||||
|
for sub in ["sidecar", "resources/sidecar", "../sidecar"] {
|
||||||
|
let cand = base.join(sub).join("detect.py");
|
||||||
|
if cand.exists() {
|
||||||
|
return Some(cand);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Dev fallback: <crate>/../sidecar/detect.py
|
||||||
|
let dev = Path::new(env!("CARGO_MANIFEST_DIR")).join("../sidecar/detect.py");
|
||||||
|
if dev.exists() {
|
||||||
|
return Some(dev);
|
||||||
|
}
|
||||||
|
None
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Build the python invocation. Prefers the Windows `py -3.10` launcher
|
||||||
|
/// (TensorFlow does not yet support 3.12+), then a venv, then plain `python`.
|
||||||
|
fn python_command(script: &Path) -> Command {
|
||||||
|
if let Ok(custom) = std::env::var("STREAM_STUDIO_PYTHON") {
|
||||||
|
let mut c = Command::new(custom);
|
||||||
|
c.arg(script);
|
||||||
|
return c;
|
||||||
|
}
|
||||||
|
#[cfg(windows)]
|
||||||
|
{
|
||||||
|
// The py launcher lets us pin 3.10 regardless of the default install.
|
||||||
|
let mut c = Command::new("py");
|
||||||
|
c.args(["-3.10"]).arg(script);
|
||||||
|
return c;
|
||||||
|
}
|
||||||
|
#[allow(unreachable_code)]
|
||||||
|
{
|
||||||
|
let mut c = Command::new("python");
|
||||||
|
c.arg(script);
|
||||||
|
c
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/// Extract 16 kHz mono WAV (the format inaSpeechSegmenter expects).
|
||||||
|
fn extract_audio(app: &AppHandle, ffmpeg: &str, video: &str, wav: &Path) -> Result<(), String> {
|
||||||
|
emit(app, "Extracting audio (16 kHz mono)…");
|
||||||
|
let mut cmd = Command::new(ffmpeg);
|
||||||
|
cmd.args([
|
||||||
|
"-y", "-i", video,
|
||||||
|
"-vn", "-acodec", "pcm_s16le", "-ar", "16000", "-ac", "1",
|
||||||
|
"-hide_banner", "-loglevel", "error",
|
||||||
|
&wav.to_string_lossy(),
|
||||||
|
]);
|
||||||
|
cmd.stdout(Stdio::null()).stderr(Stdio::piped());
|
||||||
|
hide_window(&mut cmd);
|
||||||
|
let out = cmd.output().map_err(|e| format!("ffmpeg audio extract failed: {e}"))?;
|
||||||
|
if !out.status.success() {
|
||||||
|
return Err(format!("Audio extraction failed:\n{}", String::from_utf8_lossy(&out.stderr)));
|
||||||
|
}
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn run(
|
||||||
|
app: &AppHandle,
|
||||||
|
state: &AppState,
|
||||||
|
ffmpeg: &str,
|
||||||
|
video: String,
|
||||||
|
gap: f64,
|
||||||
|
min_duration: f64,
|
||||||
|
padding: f64,
|
||||||
|
) -> Result<DetectResult, String> {
|
||||||
|
state.cancel.store(false, std::sync::atomic::Ordering::SeqCst);
|
||||||
|
|
||||||
|
let script = sidecar_script().ok_or(
|
||||||
|
"Detection sidecar not found. Expected sidecar/detect.py next to the app.",
|
||||||
|
)?;
|
||||||
|
|
||||||
|
// Temp WAV beside the source video.
|
||||||
|
let wav = Path::new(&video)
|
||||||
|
.with_extension("")
|
||||||
|
.to_string_lossy()
|
||||||
|
.to_string();
|
||||||
|
let wav = PathBuf::from(format!("{wav}.stream_studio.wav"));
|
||||||
|
|
||||||
|
extract_audio(app, ffmpeg, &video, &wav)?;
|
||||||
|
|
||||||
|
let mut cmd = python_command(&script);
|
||||||
|
cmd.args([
|
||||||
|
"--wav", &wav.to_string_lossy(),
|
||||||
|
"--gap", &gap.to_string(),
|
||||||
|
"--min-duration", &min_duration.to_string(),
|
||||||
|
"--padding", &padding.to_string(),
|
||||||
|
]);
|
||||||
|
cmd.stdout(Stdio::piped()).stderr(Stdio::piped());
|
||||||
|
cmd.env("PYTHONUNBUFFERED", "1").env("PYTHONIOENCODING", "utf-8");
|
||||||
|
hide_window(&mut cmd);
|
||||||
|
|
||||||
|
emit(app, "Loading detection model…");
|
||||||
|
let mut child = cmd.spawn().map_err(|e| {
|
||||||
|
format!("Could not start Python sidecar ({e}). Is Python 3.10 + inaSpeechSegmenter installed?")
|
||||||
|
})?;
|
||||||
|
let stdout = child.stdout.take().ok_or("No sidecar stdout")?;
|
||||||
|
let stderr = child.stderr.take();
|
||||||
|
*state.proc.lock().unwrap() = Some(child);
|
||||||
|
|
||||||
|
let mut result: Option<DetectResult> = None;
|
||||||
|
let mut err_msg: Option<String> = None;
|
||||||
|
|
||||||
|
for line in BufReader::new(stdout).lines() {
|
||||||
|
let line = match line {
|
||||||
|
Ok(l) => l,
|
||||||
|
Err(_) => break,
|
||||||
|
};
|
||||||
|
let line = line.trim();
|
||||||
|
if line.is_empty() {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
let parsed: Value = match serde_json::from_str(line) {
|
||||||
|
Ok(v) => v,
|
||||||
|
Err(_) => continue, // ignore stray non-JSON output
|
||||||
|
};
|
||||||
|
match parsed.get("type").and_then(|t| t.as_str()) {
|
||||||
|
Some("progress") => {
|
||||||
|
if let Some(m) = parsed.get("message").and_then(|m| m.as_str()) {
|
||||||
|
emit(app, m);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Some("result") => {
|
||||||
|
let segments: Vec<Segment> = serde_json::from_value(
|
||||||
|
parsed.get("segments").cloned().unwrap_or(Value::Array(vec![])),
|
||||||
|
)
|
||||||
|
.unwrap_or_default();
|
||||||
|
let stats = parsed.get("stats").cloned().unwrap_or(Value::Null);
|
||||||
|
result = Some(DetectResult { segments, stats });
|
||||||
|
}
|
||||||
|
Some("error") => {
|
||||||
|
err_msg = parsed
|
||||||
|
.get("message")
|
||||||
|
.and_then(|m| m.as_str())
|
||||||
|
.map(|s| s.to_string());
|
||||||
|
}
|
||||||
|
_ => {}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
let mut child = state.proc.lock().unwrap().take().ok_or("Sidecar vanished")?;
|
||||||
|
let status = child.wait().map_err(|e| format!("sidecar wait failed: {e}"))?;
|
||||||
|
let mut stderr_text = String::new();
|
||||||
|
if let Some(mut se) = stderr {
|
||||||
|
use std::io::Read;
|
||||||
|
let _ = se.read_to_string(&mut stderr_text);
|
||||||
|
}
|
||||||
|
|
||||||
|
let _ = std::fs::remove_file(&wav);
|
||||||
|
|
||||||
|
if state.cancel.load(std::sync::atomic::Ordering::SeqCst) {
|
||||||
|
return Err("__cancelled__".into());
|
||||||
|
}
|
||||||
|
if let Some(m) = err_msg {
|
||||||
|
return Err(m);
|
||||||
|
}
|
||||||
|
if let Some(r) = result {
|
||||||
|
return Ok(r);
|
||||||
|
}
|
||||||
|
if !status.success() {
|
||||||
|
let tail: String = stderr_text.chars().rev().take(800).collect::<String>().chars().rev().collect();
|
||||||
|
return Err(format!("Detection failed.\n{tail}"));
|
||||||
|
}
|
||||||
|
Err("Detection produced no result.".into())
|
||||||
|
}
|
||||||
129
detect.ts
Normal file
129
detect.ts
Normal file
@@ -0,0 +1,129 @@
|
|||||||
|
// Detect view — find singing segments in one video, export editor markers.
|
||||||
|
|
||||||
|
import { h, clear, basename, fmtDur } from "../dom";
|
||||||
|
import { makeProgress, toast, renderSongs } from "../ui";
|
||||||
|
import {
|
||||||
|
pickVideo, detectSongs, cancel, exportSegments, onDetectProgress, reveal,
|
||||||
|
type Segment,
|
||||||
|
} from "../api";
|
||||||
|
|
||||||
|
export interface View { el: HTMLElement; onDrop: (paths: string[]) => void; setVideo: (p: string) => void; }
|
||||||
|
|
||||||
|
const FPS = ["29.97", "25", "23.976", "30", "60"];
|
||||||
|
|
||||||
|
function numField(label: string, value: number, step = "1"): [HTMLElement, HTMLInputElement] {
|
||||||
|
const input = h("input", { class: "input", type: "number", value: String(value), step, min: "0" }) as HTMLInputElement;
|
||||||
|
return [h("div", { class: "field" }, h("label", {}, label), input), input];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function buildDetect(): View {
|
||||||
|
let running = false;
|
||||||
|
let segments: Segment[] = [];
|
||||||
|
let videoPath = "";
|
||||||
|
|
||||||
|
const videoInput = h("input", { class: "input grow", placeholder: "Select a video…", readonly: true }) as HTMLInputElement;
|
||||||
|
const [gapF, gapI] = numField("Merge gap (s)", 10);
|
||||||
|
const [minF, minI] = numField("Min duration (s)", 20);
|
||||||
|
const [padF, padI] = numField("Padding (s)", 2);
|
||||||
|
const fpsSel = h("select", { class: "input" }, ...FPS.map((f) => h("option", { value: f }, f))) as HTMLSelectElement;
|
||||||
|
|
||||||
|
const detectBtn = h("button", { class: "btn primary lg", disabled: true }, "🎤 Detect Songs") as HTMLButtonElement;
|
||||||
|
const stopBtn = h("button", { class: "btn danger", disabled: true }, "■ Stop") as HTMLButtonElement;
|
||||||
|
const exportBtn = h("button", { class: "btn", disabled: true }, "⬇ Export EDL / CSV") as HTMLButtonElement;
|
||||||
|
const revealBtn = h("button", { class: "btn ghost", style: "display:none" }, "Show in folder") as HTMLButtonElement;
|
||||||
|
const progress = makeProgress();
|
||||||
|
const statsEl = h("div", { class: "pills" });
|
||||||
|
const songsEl = h("div", { class: "songs" });
|
||||||
|
let lastExportDir = "";
|
||||||
|
|
||||||
|
function setVideo(p: string) {
|
||||||
|
videoPath = p;
|
||||||
|
videoInput.value = basename(p);
|
||||||
|
videoInput.title = p;
|
||||||
|
detectBtn.disabled = running || !p;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function detect() {
|
||||||
|
if (!videoPath) return;
|
||||||
|
running = true;
|
||||||
|
detectBtn.disabled = true;
|
||||||
|
stopBtn.disabled = false;
|
||||||
|
exportBtn.disabled = true;
|
||||||
|
revealBtn.style.display = "none";
|
||||||
|
clear(songsEl); clear(statsEl);
|
||||||
|
progress.reset();
|
||||||
|
progress.indeterminate(true, "Starting detection…");
|
||||||
|
|
||||||
|
const un = await onDetectProgress((msg) => {
|
||||||
|
progress.indeterminate(true, msg);
|
||||||
|
progress.appendLog(msg);
|
||||||
|
});
|
||||||
|
|
||||||
|
try {
|
||||||
|
const res = await detectSongs(videoPath, +gapI.value, +minI.value, +padI.value);
|
||||||
|
segments = res.segments;
|
||||||
|
progress.indeterminate(false);
|
||||||
|
progress.set(1, `Found ${segments.length} singing segment(s).`);
|
||||||
|
renderSongs(songsEl, segments);
|
||||||
|
const totalSing = segments.reduce((a, s) => a + s.duration, 0);
|
||||||
|
statsEl.replaceChildren(
|
||||||
|
h("span", { class: "pill" }, `${segments.length} songs`),
|
||||||
|
h("span", { class: "pill" }, `singing ${fmtDur(totalSing)}`),
|
||||||
|
...Object.entries(res.stats || {}).map(([k, v]) =>
|
||||||
|
h("span", { class: "pill" }, `${k} ${fmtDur(v as number)}`)),
|
||||||
|
);
|
||||||
|
exportBtn.disabled = segments.length === 0;
|
||||||
|
if (!segments.length) toast("No singing detected — try a larger gap or smaller min duration.", "");
|
||||||
|
} catch (e) {
|
||||||
|
const msg = String(e);
|
||||||
|
progress.indeterminate(false);
|
||||||
|
if (msg.includes("__cancelled__")) progress.set(0, "Stopped by user.");
|
||||||
|
else { progress.set(0, "Detection failed."); toast(msg, "err"); }
|
||||||
|
} finally {
|
||||||
|
un();
|
||||||
|
running = false;
|
||||||
|
stopBtn.disabled = true;
|
||||||
|
detectBtn.disabled = !videoPath;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
exportBtn.addEventListener("click", async () => {
|
||||||
|
if (!segments.length) return;
|
||||||
|
try {
|
||||||
|
const res = await exportSegments(segments, videoPath, null, +fpsSel.value);
|
||||||
|
lastExportDir = res.files[0] ?? "";
|
||||||
|
toast(`Exported ${res.files.length} files`, "ok");
|
||||||
|
if (lastExportDir) revealBtn.style.display = "";
|
||||||
|
} catch (e) { toast(String(e), "err"); }
|
||||||
|
});
|
||||||
|
revealBtn.addEventListener("click", () => lastExportDir && reveal(lastExportDir));
|
||||||
|
detectBtn.addEventListener("click", detect);
|
||||||
|
stopBtn.addEventListener("click", () => cancel());
|
||||||
|
|
||||||
|
const el = h("div", { class: "view" },
|
||||||
|
h("div", { class: "card" },
|
||||||
|
h("h2", {}, h("span", { class: "step" }, "1"), "Source Video"),
|
||||||
|
h("div", { class: "row" }, videoInput,
|
||||||
|
h("button", { class: "btn", onclick: async () => { const p = await pickVideo(); if (p) setVideo(p); } }, "Browse…")),
|
||||||
|
h("div", { class: "hint", style: "margin-top:8px" }, "Tip: drop a file here, or use a Concat output."),
|
||||||
|
),
|
||||||
|
h("div", { class: "card" },
|
||||||
|
h("h2", {}, h("span", { class: "step" }, "2"), "Detection Settings"),
|
||||||
|
h("div", { class: "row wrap" },
|
||||||
|
h("div", { class: "grow", style: "min-width:130px" }, gapF),
|
||||||
|
h("div", { class: "grow", style: "min-width:130px" }, minF),
|
||||||
|
h("div", { class: "grow", style: "min-width:130px" }, padF),
|
||||||
|
h("div", { class: "field", style: "min-width:120px" }, h("label", {}, "FPS (export)"), fpsSel)),
|
||||||
|
h("div", { class: "hint", style: "margin-top:8px" }, "Singing = \"music\". More chatting between songs → raise the gap. Short covers → lower the min duration."),
|
||||||
|
),
|
||||||
|
h("div", { class: "card" },
|
||||||
|
h("h2", {}, h("span", { class: "step" }, "3"), "Run"),
|
||||||
|
h("div", { class: "row", style: "margin-bottom:14px" }, detectBtn, stopBtn, exportBtn, revealBtn),
|
||||||
|
progress.root,
|
||||||
|
h("div", { style: "margin-top:12px" }, statsEl),
|
||||||
|
h("div", { style: "margin-top:10px" }, songsEl),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
return { el, onDrop: (paths) => { if (paths[0]) setVideo(paths[0]); }, setVideo };
|
||||||
|
}
|
||||||
114
sidecar_detect.py
Normal file
114
sidecar_detect.py
Normal file
@@ -0,0 +1,114 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Stream Studio — singing-detection sidecar.
|
||||||
|
|
||||||
|
This is the ONLY Python in the project. It exists solely because the ML model
|
||||||
|
(inaSpeechSegmenter, which classifies singing voice as "music") is TensorFlow-
|
||||||
|
based and has no native Rust/JS equivalent. Everything else — UI, ffmpeg
|
||||||
|
concat, audio extraction, file exports — is handled by the Tauri/Rust host.
|
||||||
|
|
||||||
|
Contract (line-delimited JSON on stdout):
|
||||||
|
{"type": "progress", "message": "..."}
|
||||||
|
{"type": "result", "segments": [{"index","start","end","duration"}...],
|
||||||
|
"stats": {"music": 123.4, "speech": 456.7, ...}}
|
||||||
|
{"type": "error", "message": "..."}
|
||||||
|
|
||||||
|
The host extracts a 16 kHz mono WAV and passes it via --wav, so this script
|
||||||
|
never touches ffmpeg.
|
||||||
|
|
||||||
|
Run with Python 3.10 (TensorFlow does not support 3.12+):
|
||||||
|
py -3.10 detect.py --wav audio.wav --gap 10 --min-duration 20 --padding 2
|
||||||
|
"""
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import json
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
|
||||||
|
|
||||||
|
def emit(obj: dict) -> None:
|
||||||
|
"""Write one JSON record per line and flush immediately."""
|
||||||
|
sys.stdout.write(json.dumps(obj, ensure_ascii=False) + "\n")
|
||||||
|
sys.stdout.flush()
|
||||||
|
|
||||||
|
|
||||||
|
def progress(msg: str) -> None:
|
||||||
|
emit({"type": "progress", "message": msg})
|
||||||
|
|
||||||
|
|
||||||
|
def merge_singing_segments(segments, max_gap, min_duration, padding):
|
||||||
|
"""Filter for 'music' segments and merge nearby ones into songs."""
|
||||||
|
music = sorted((s, e) for label, s, e in segments if label == "music")
|
||||||
|
if not music:
|
||||||
|
return []
|
||||||
|
|
||||||
|
merged = []
|
||||||
|
cur_start, cur_end = music[0]
|
||||||
|
for s, e in music[1:]:
|
||||||
|
if s - cur_end <= max_gap:
|
||||||
|
cur_end = max(cur_end, e)
|
||||||
|
else:
|
||||||
|
merged.append((cur_start, cur_end))
|
||||||
|
cur_start, cur_end = s, e
|
||||||
|
merged.append((cur_start, cur_end))
|
||||||
|
|
||||||
|
results = []
|
||||||
|
idx = 1
|
||||||
|
for s, e in merged:
|
||||||
|
if (e - s) >= min_duration:
|
||||||
|
ps = max(0.0, s - padding)
|
||||||
|
pe = e + padding
|
||||||
|
results.append({
|
||||||
|
"index": idx,
|
||||||
|
"start": round(ps, 3),
|
||||||
|
"end": round(pe, 3),
|
||||||
|
"duration": round(pe - ps, 3),
|
||||||
|
})
|
||||||
|
idx += 1
|
||||||
|
return results
|
||||||
|
|
||||||
|
|
||||||
|
def main() -> int:
|
||||||
|
ap = argparse.ArgumentParser()
|
||||||
|
ap.add_argument("--wav", required=True)
|
||||||
|
ap.add_argument("--gap", type=float, default=10.0)
|
||||||
|
ap.add_argument("--min-duration", type=float, default=20.0)
|
||||||
|
ap.add_argument("--padding", type=float, default=2.0)
|
||||||
|
args = ap.parse_args()
|
||||||
|
|
||||||
|
try:
|
||||||
|
progress("Loading detection model (first run downloads weights)…")
|
||||||
|
from inaSpeechSegmenter import Segmenter
|
||||||
|
except ImportError:
|
||||||
|
emit({
|
||||||
|
"type": "error",
|
||||||
|
"message": (
|
||||||
|
"inaSpeechSegmenter is not installed for this Python.\n"
|
||||||
|
"Install it with:\n"
|
||||||
|
" py -3.10 -m pip install inaSpeechSegmenter tensorflow"
|
||||||
|
),
|
||||||
|
})
|
||||||
|
return 1
|
||||||
|
|
||||||
|
try:
|
||||||
|
t0 = time.time()
|
||||||
|
seg = Segmenter(vad_engine="smn", detect_gender=False)
|
||||||
|
progress("Analysing audio — this can take a while on long streams…")
|
||||||
|
raw = seg(args.wav)
|
||||||
|
progress(f"Segmentation done in {time.time() - t0:.0f}s ({len(raw)} raw segments).")
|
||||||
|
|
||||||
|
stats = {}
|
||||||
|
for label, s, e in raw:
|
||||||
|
stats[label] = round(stats.get(label, 0.0) + (e - s), 1)
|
||||||
|
|
||||||
|
songs = merge_singing_segments(raw, args.gap, args.min_duration, args.padding)
|
||||||
|
progress(f"Found {len(songs)} singing segment(s) after merge + filter.")
|
||||||
|
emit({"type": "result", "segments": songs, "stats": stats})
|
||||||
|
return 0
|
||||||
|
except Exception as exc: # noqa: BLE001 — surface any model/runtime error to the host
|
||||||
|
emit({"type": "error", "message": f"{type(exc).__name__}: {exc}"})
|
||||||
|
return 1
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
sys.exit(main())
|
||||||
261
sing_detect.py
261
sing_detect.py
@@ -1,261 +0,0 @@
|
|||||||
import customtkinter as ctk
|
|
||||||
import tkinter as tk
|
|
||||||
from tkinter import filedialog, messagebox
|
|
||||||
import os
|
|
||||||
import csv
|
|
||||||
from datetime import timedelta
|
|
||||||
|
|
||||||
import math
|
|
||||||
import librosa
|
|
||||||
import soundfile as sf
|
|
||||||
|
|
||||||
from transformers import pipeline
|
|
||||||
from pyannote.core import Segment
|
|
||||||
|
|
||||||
# ------------------------
|
|
||||||
# Time formatting function
|
|
||||||
# ------------------------
|
|
||||||
def seconds_to_timecode(total_seconds):
|
|
||||||
"""
|
|
||||||
Convert float seconds -> "HH:MM:SS.mmm" for DaVinci Resolve CSV markers.
|
|
||||||
"""
|
|
||||||
td = timedelta(seconds=total_seconds)
|
|
||||||
hours, remainder = divmod(td.seconds, 3600)
|
|
||||||
minutes, secs = divmod(remainder, 60)
|
|
||||||
ms = int(td.microseconds / 1000)
|
|
||||||
return f"{hours:02d}:{minutes:02d}:{secs:02d}.{ms:03d}"
|
|
||||||
|
|
||||||
def chunk_audio(audio_path, chunk_length_s=10.0, stride_s=5.0, sr=16000):
|
|
||||||
"""
|
|
||||||
Loads the entire audio with librosa, then yields overlapping chunks
|
|
||||||
(chunk_data, chunk_start, chunk_end, sr).
|
|
||||||
|
|
||||||
- chunk_length_s = length (seconds) of each chunk
|
|
||||||
- stride_s = how much we "step back" within each chunk for overlap
|
|
||||||
e.g. chunk_length=10, stride=5 => chunk #0 covers
|
|
||||||
0-10s, chunk #1 covers 5-15s, chunk #2 covers 10-20s, etc.
|
|
||||||
- sr = sampling rate to load the audio
|
|
||||||
"""
|
|
||||||
audio, sr = librosa.load(audio_path, sr=sr)
|
|
||||||
total_len_s = len(audio) / sr
|
|
||||||
|
|
||||||
# Step in seconds, between the start points of consecutive chunks
|
|
||||||
# e.g. for chunk=10, stride=5 => step=10 - 5=5
|
|
||||||
# means next chunk starts 5s after the previous chunk start
|
|
||||||
step = chunk_length_s - stride_s
|
|
||||||
|
|
||||||
# Edge case: if stride_s >= chunk_length_s, you won't have overlap
|
|
||||||
if step <= 0:
|
|
||||||
step = chunk_length_s # no overlap
|
|
||||||
|
|
||||||
# Figure out how many chunks needed so we don’t exceed total length
|
|
||||||
# (a bit of math to ensure we handle trailing audio < chunk_length_s)
|
|
||||||
num_chunks = math.ceil((total_len_s - chunk_length_s) / step) + 1
|
|
||||||
if num_chunks < 1:
|
|
||||||
num_chunks = 1
|
|
||||||
|
|
||||||
for i in range(num_chunks):
|
|
||||||
chunk_start = i * step
|
|
||||||
chunk_end = chunk_start + chunk_length_s
|
|
||||||
|
|
||||||
# If the chunk goes beyond total_len_s, clamp it
|
|
||||||
if chunk_end > total_len_s:
|
|
||||||
chunk_end = total_len_s
|
|
||||||
|
|
||||||
# Convert times to sample indexes
|
|
||||||
start_sample = int(chunk_start * sr)
|
|
||||||
end_sample = int(chunk_end * sr)
|
|
||||||
|
|
||||||
# If our chunk_start >= total_len_s, we can stop
|
|
||||||
if chunk_start >= total_len_s:
|
|
||||||
break
|
|
||||||
|
|
||||||
# Slice the waveform
|
|
||||||
chunk_data = audio[start_sample:end_sample]
|
|
||||||
|
|
||||||
yield chunk_data, chunk_start, chunk_end, sr
|
|
||||||
|
|
||||||
class MusicMarkerApp(ctk.CTk):
|
|
||||||
def __init__(self):
|
|
||||||
super().__init__()
|
|
||||||
|
|
||||||
self.title("Music Marker Generator")
|
|
||||||
self.geometry("500x350")
|
|
||||||
|
|
||||||
# Variables to hold file paths & threshold
|
|
||||||
self.audio_file_path = tk.StringVar(value="")
|
|
||||||
self.output_csv_path = tk.StringVar(value="")
|
|
||||||
self.merge_threshold_var = tk.DoubleVar(value=2.0) # default: merge segments within 2 seconds
|
|
||||||
self.pipeline = None # We'll load it once on demand
|
|
||||||
|
|
||||||
self._create_widgets()
|
|
||||||
|
|
||||||
def _create_widgets(self):
|
|
||||||
# 1) Frame for selecting audio file
|
|
||||||
file_frame = ctk.CTkFrame(self)
|
|
||||||
file_frame.pack(pady=10, padx=10, fill="x")
|
|
||||||
|
|
||||||
file_label = ctk.CTkLabel(file_frame, text="Audio File:")
|
|
||||||
file_label.pack(side="left", padx=5)
|
|
||||||
|
|
||||||
file_entry = ctk.CTkEntry(file_frame, textvariable=self.audio_file_path, width=250)
|
|
||||||
file_entry.pack(side="left", padx=5)
|
|
||||||
|
|
||||||
file_button = ctk.CTkButton(file_frame, text="Browse", command=self._browse_audio_file)
|
|
||||||
file_button.pack(side="left", padx=5)
|
|
||||||
|
|
||||||
# 2) Frame for output CSV
|
|
||||||
output_frame = ctk.CTkFrame(self)
|
|
||||||
output_frame.pack(pady=10, padx=10, fill="x")
|
|
||||||
|
|
||||||
output_label = ctk.CTkLabel(output_frame, text="Output CSV:")
|
|
||||||
output_label.pack(side="left", padx=5)
|
|
||||||
|
|
||||||
output_entry = ctk.CTkEntry(output_frame, textvariable=self.output_csv_path, width=250)
|
|
||||||
output_entry.pack(side="left", padx=5)
|
|
||||||
|
|
||||||
output_button = ctk.CTkButton(output_frame, text="Browse", command=self._browse_output_csv)
|
|
||||||
output_button.pack(side="left", padx=5)
|
|
||||||
|
|
||||||
# 3) Merge Threshold
|
|
||||||
threshold_frame = ctk.CTkFrame(self)
|
|
||||||
threshold_frame.pack(pady=10, padx=10, fill="x")
|
|
||||||
|
|
||||||
threshold_label = ctk.CTkLabel(threshold_frame, text="Merge Gap (sec):")
|
|
||||||
threshold_label.pack(side="left", padx=5)
|
|
||||||
|
|
||||||
threshold_entry = ctk.CTkEntry(threshold_frame, textvariable=self.merge_threshold_var, width=50)
|
|
||||||
threshold_entry.pack(side="left", padx=5)
|
|
||||||
|
|
||||||
# 4) Run button
|
|
||||||
run_button = ctk.CTkButton(self, text="Run Music Detection", command=self._run_detection)
|
|
||||||
run_button.pack(pady=10)
|
|
||||||
|
|
||||||
# 5) Status label
|
|
||||||
self.status_label = ctk.CTkLabel(self, text="", wraplength=400, justify="left")
|
|
||||||
self.status_label.pack(pady=5)
|
|
||||||
|
|
||||||
def _browse_audio_file(self):
|
|
||||||
file_path = filedialog.askopenfilename(
|
|
||||||
title="Select Audio File",
|
|
||||||
filetypes=[("Audio Files", "*.wav *.mp3 *.flac *.m4a *.aac *.ogg *.wma *.aif *.aiff")]
|
|
||||||
)
|
|
||||||
if file_path:
|
|
||||||
self.audio_file_path.set(file_path)
|
|
||||||
|
|
||||||
def _browse_output_csv(self):
|
|
||||||
file_path = filedialog.asksaveasfilename(
|
|
||||||
title="Select Output CSV",
|
|
||||||
defaultextension=".csv",
|
|
||||||
filetypes=[("CSV Files", "*.csv")]
|
|
||||||
)
|
|
||||||
if file_path:
|
|
||||||
self.output_csv_path.set(file_path)
|
|
||||||
|
|
||||||
def _run_detection(self):
|
|
||||||
audio_path = self.audio_file_path.get().strip()
|
|
||||||
output_csv = self.output_csv_path.get().strip()
|
|
||||||
merge_threshold = self.merge_threshold_var.get()
|
|
||||||
|
|
||||||
if not audio_path or not os.path.isfile(audio_path):
|
|
||||||
messagebox.showerror("Error", "Please select a valid audio file.")
|
|
||||||
return
|
|
||||||
if not output_csv:
|
|
||||||
messagebox.showerror("Error", "Please specify an output CSV file.")
|
|
||||||
return
|
|
||||||
|
|
||||||
self._set_status("Loading model, please wait...")
|
|
||||||
|
|
||||||
# Load pipeline if not loaded yet
|
|
||||||
if self.pipeline is None:
|
|
||||||
try:
|
|
||||||
# Standard audio-classification pipeline
|
|
||||||
self.pipeline = pipeline(
|
|
||||||
"audio-classification",
|
|
||||||
model="MarekCech/GenreVim-Music-Detection-DistilHuBERT"
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
messagebox.showerror("Model Error", f"Could not load the model:\n{e}")
|
|
||||||
return
|
|
||||||
|
|
||||||
self._set_status("Chunking audio and running music detection...")
|
|
||||||
|
|
||||||
# We'll collect "music" segments from each chunk
|
|
||||||
music_segments = []
|
|
||||||
|
|
||||||
# Manually chunk the audio & classify each chunk
|
|
||||||
try:
|
|
||||||
for chunk_data, chunk_start, chunk_end, sr in chunk_audio(
|
|
||||||
audio_path,
|
|
||||||
chunk_length_s=10.0,
|
|
||||||
stride_s=5.0,
|
|
||||||
sr=16000
|
|
||||||
):
|
|
||||||
# The pipeline expects a waveform plus sampling_rate, not a path
|
|
||||||
# So pass the chunk_data + sr
|
|
||||||
results = self.pipeline(chunk_data, sampling_rate=sr)
|
|
||||||
if not results:
|
|
||||||
continue
|
|
||||||
|
|
||||||
# best guess from the pipeline for this chunk
|
|
||||||
best_guess = max(results, key=lambda x: x["score"])
|
|
||||||
if best_guess["label"].lower() == "music" and best_guess["score"] > 0.5:
|
|
||||||
music_segments.append(Segment(chunk_start, chunk_end))
|
|
||||||
|
|
||||||
except Exception as e:
|
|
||||||
messagebox.showerror("Detection Error", f"Error running music detection:\n{e}")
|
|
||||||
return
|
|
||||||
|
|
||||||
# Sort segments by start time
|
|
||||||
music_segments.sort(key=lambda seg: seg.start)
|
|
||||||
|
|
||||||
# Merge close segments based on threshold
|
|
||||||
merged_segments = []
|
|
||||||
if not music_segments:
|
|
||||||
self._set_status("No music segments found. No CSV generated.")
|
|
||||||
return
|
|
||||||
else:
|
|
||||||
current_start = music_segments[0].start
|
|
||||||
current_end = music_segments[0].end
|
|
||||||
|
|
||||||
for seg in music_segments[1:]:
|
|
||||||
# If the new segment starts within X seconds of the old segment, merge them
|
|
||||||
if seg.start <= current_end + merge_threshold:
|
|
||||||
current_end = max(current_end, seg.end)
|
|
||||||
else:
|
|
||||||
merged_segments.append(Segment(current_start, current_end))
|
|
||||||
current_start = seg.start
|
|
||||||
current_end = seg.end
|
|
||||||
|
|
||||||
# finalize last
|
|
||||||
merged_segments.append(Segment(current_start, current_end))
|
|
||||||
|
|
||||||
# Write CSV for DaVinci Resolve
|
|
||||||
try:
|
|
||||||
with open(output_csv, "w", newline="", encoding="utf-8") as csvfile:
|
|
||||||
writer = csv.writer(csvfile)
|
|
||||||
# DaVinci Resolve CSV marker columns
|
|
||||||
writer.writerow(["Name", "Start", "End", "Color", "Marker Type"])
|
|
||||||
|
|
||||||
for i, seg in enumerate(merged_segments, start=1):
|
|
||||||
start_tc = seconds_to_timecode(seg.start)
|
|
||||||
end_tc = seconds_to_timecode(seg.end)
|
|
||||||
name = f"Music Segment #{i}"
|
|
||||||
color = "Green"
|
|
||||||
marker_type = "Comment"
|
|
||||||
writer.writerow([name, start_tc, end_tc, color, marker_type])
|
|
||||||
|
|
||||||
self._set_status(
|
|
||||||
f"Done! Found {len(merged_segments)} music segments.\n"
|
|
||||||
f"CSV saved to: {output_csv}"
|
|
||||||
)
|
|
||||||
except Exception as e:
|
|
||||||
messagebox.showerror("File Error", f"Could not write CSV:\n{e}")
|
|
||||||
|
|
||||||
def _set_status(self, msg):
|
|
||||||
self.status_label.configure(text=msg)
|
|
||||||
self.update_idletasks()
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
app = MusicMarkerApp()
|
|
||||||
app.mainloop()
|
|
||||||
Reference in New Issue
Block a user