Compare commits
1 Commits
ten-vad
...
feat/ten-v
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
92f0cf2785 |
@@ -31,7 +31,6 @@ interface Props {
|
|||||||
max: number;
|
max: number;
|
||||||
step: number;
|
step: number;
|
||||||
disabled?: boolean;
|
disabled?: boolean;
|
||||||
tooltip?: boolean;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -47,7 +46,6 @@ export const Slider: FC<Props> = ({
|
|||||||
max,
|
max,
|
||||||
step,
|
step,
|
||||||
disabled,
|
disabled,
|
||||||
tooltip = true,
|
|
||||||
}) => {
|
}) => {
|
||||||
const onValueChange = useCallback(
|
const onValueChange = useCallback(
|
||||||
([v]: number[]) => onValueChangeProp(v),
|
([v]: number[]) => onValueChangeProp(v),
|
||||||
@@ -73,13 +71,9 @@ export const Slider: FC<Props> = ({
|
|||||||
<Range className={styles.highlight} />
|
<Range className={styles.highlight} />
|
||||||
</Track>
|
</Track>
|
||||||
{/* Note: This is expected not to be visible on mobile.*/}
|
{/* Note: This is expected not to be visible on mobile.*/}
|
||||||
{tooltip ? (
|
<Tooltip placement="top" label={Math.round(value * 100).toString() + "%"}>
|
||||||
<Tooltip placement="top" label={Math.round(value * 100).toString() + "%"}>
|
|
||||||
<Thumb className={styles.handle} aria-label={label} />
|
|
||||||
</Tooltip>
|
|
||||||
) : (
|
|
||||||
<Thumb className={styles.handle} aria-label={label} />
|
<Thumb className={styles.handle} aria-label={label} />
|
||||||
)}
|
</Tooltip>
|
||||||
</Root>
|
</Root>
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -74,8 +74,6 @@ export function LivekitRoomAudioRenderer({
|
|||||||
)
|
)
|
||||||
// Only keep audio tracks
|
// Only keep audio tracks
|
||||||
.filter((ref) => ref.publication.kind === Track.Kind.Audio)
|
.filter((ref) => ref.publication.kind === Track.Kind.Audio)
|
||||||
// Never render local participant's own audio back to themselves
|
|
||||||
.filter((ref) => !ref.participant.isLocal)
|
|
||||||
// Only keep tracks from participants that are in the validIdentities list
|
// Only keep tracks from participants that are in the validIdentities list
|
||||||
.filter((ref) => {
|
.filter((ref) => {
|
||||||
const isValid = validIdentities.includes(ref.participant.identity);
|
const isValid = validIdentities.includes(ref.participant.identity);
|
||||||
|
|||||||
@@ -23,19 +23,13 @@ declare function registerProcessor(
|
|||||||
processorCtor: new (...args: any[]) => AudioWorkletProcessor,
|
processorCtor: new (...args: any[]) => AudioWorkletProcessor,
|
||||||
): void;
|
): void;
|
||||||
|
|
||||||
interface NoiseGateParams {
|
interface TenVadParams {
|
||||||
noiseGateActive: boolean;
|
|
||||||
threshold: number; // dBFS — gate opens above this, closes below it
|
|
||||||
attackMs: number;
|
|
||||||
holdMs: number;
|
|
||||||
releaseMs: number;
|
|
||||||
transientEnabled: boolean;
|
|
||||||
transientThresholdDb: number; // dB above background RMS that triggers suppression
|
|
||||||
transientReleaseMs: number; // how quickly suppression fades after transient ends
|
|
||||||
// TEN-VAD params
|
// TEN-VAD params
|
||||||
vadEnabled: boolean;
|
vadEnabled: boolean;
|
||||||
vadPositiveThreshold: number; // open gate when prob >= this (0–1)
|
vadPositiveThreshold: number; // open gate when prob >= this (0–1)
|
||||||
vadNegativeThreshold: number; // close gate when prob < this (0–1)
|
vadNegativeThreshold: number; // close gate when prob < this (0–1)
|
||||||
|
vadMode: "standard" | "aggressive" | "loose";
|
||||||
|
holdMs: number; // hold time before closing gate (ms); 0 = no hold
|
||||||
}
|
}
|
||||||
|
|
||||||
interface VADGateMessage {
|
interface VADGateMessage {
|
||||||
@@ -43,10 +37,6 @@ interface VADGateMessage {
|
|||||||
open: boolean;
|
open: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
function dbToLinear(db: number): number {
|
|
||||||
return Math.pow(10, db / 20);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Thin synchronous wrapper around the TEN-VAD Emscripten WASM module.
|
* Thin synchronous wrapper around the TEN-VAD Emscripten WASM module.
|
||||||
* Instantiated synchronously in the AudioWorklet constructor from a
|
* Instantiated synchronously in the AudioWorklet constructor from a
|
||||||
@@ -170,55 +160,39 @@ class TenVADRuntime {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* AudioWorkletProcessor implementing a noise gate, an optional transient
|
* AudioWorkletProcessor implementing an in-worklet TEN-VAD gate running
|
||||||
* suppressor, and an optional in-worklet TEN-VAD gate — all running
|
* per-sample.
|
||||||
* per-sample in a single pass.
|
|
||||||
*
|
|
||||||
* Noise gate: opens when instantaneous peak exceeds threshold, closes below.
|
|
||||||
* Attack, hold, and release times smooth the attenuation envelope.
|
|
||||||
*
|
|
||||||
* Transient suppressor: tracks a slow-moving RMS background level. When the
|
|
||||||
* instantaneous peak exceeds the background by more than transientThresholdDb,
|
|
||||||
* gain is instantly cut to 0 and releases over transientReleaseMs.
|
|
||||||
*
|
*
|
||||||
* TEN-VAD gate: accumulates audio with 3:1 decimation (48 kHz → 16 kHz),
|
* TEN-VAD gate: accumulates audio with 3:1 decimation (48 kHz → 16 kHz),
|
||||||
* runs the TEN-VAD model synchronously every 256 samples (16 ms), and
|
* runs the TEN-VAD model synchronously every 256 samples (16 ms), and
|
||||||
* controls vadGateOpen with hysteresis. No IPC round-trip required.
|
* controls vadGateOpen with hysteresis. No IPC round-trip required.
|
||||||
|
* Asymmetric ramp: 5 ms open (minimise speech onset masking), 20 ms close
|
||||||
|
* (de-click on silence).
|
||||||
*/
|
*/
|
||||||
class NoiseGateProcessor extends AudioWorkletProcessor {
|
class TenVadProcessor extends AudioWorkletProcessor {
|
||||||
// Noise gate state
|
|
||||||
private noiseGateActive = true;
|
|
||||||
private threshold = dbToLinear(-60);
|
|
||||||
private attackRate = 1.0 / (0.025 * sampleRate);
|
|
||||||
private releaseRate = 1.0 / (0.15 * sampleRate);
|
|
||||||
private holdTime = 0.2;
|
|
||||||
private isOpen = false;
|
|
||||||
private gateAttenuation = 0;
|
|
||||||
private heldTime = 0;
|
|
||||||
|
|
||||||
// Transient suppressor state
|
|
||||||
private transientEnabled = false;
|
|
||||||
private transientRatio = dbToLinear(15);
|
|
||||||
private transientReleaseRate = 1.0 / (0.08 * sampleRate);
|
|
||||||
private transientAttenuation = 1.0;
|
|
||||||
private slowRms = 0;
|
|
||||||
private rmsCoeff = Math.exp(-1.0 / (0.2 * sampleRate));
|
|
||||||
|
|
||||||
// VAD gate state
|
// VAD gate state
|
||||||
private vadGateOpen = true; // starts open; TEN-VAD closes it on first silent frame
|
private vadGateOpen = true; // starts open; TEN-VAD closes it on first silent frame
|
||||||
private vadAttenuation = 1.0;
|
private vadAttenuation = 1.0;
|
||||||
private readonly vadRampRate = 1.0 / (0.02 * sampleRate);
|
// Asymmetric ramp rates — recomputed in updateParams based on vadAggressive
|
||||||
|
private vadOpenRampRate = 1.0 / (0.005 * sampleRate); // default: 5 ms
|
||||||
|
private vadCloseRampRate = 1.0 / (0.02 * sampleRate); // default: 20 ms
|
||||||
|
|
||||||
// TEN-VAD state
|
// TEN-VAD state
|
||||||
private vadEnabled = false;
|
private vadEnabled = false;
|
||||||
private vadPositiveThreshold = 0.5;
|
private vadPositiveThreshold = 0.5;
|
||||||
private vadNegativeThreshold = 0.3;
|
private vadNegativeThreshold = 0.3;
|
||||||
|
private holdMs = 0;
|
||||||
|
private vadHoldHops = 0; // hold expressed in VAD hops
|
||||||
|
private vadHoldCounter = 0; // hops of continuous sub-threshold signal while gate is open
|
||||||
private tenVadRuntime: TenVADRuntime | null = null;
|
private tenVadRuntime: TenVADRuntime | null = null;
|
||||||
|
private tenVadModule: WebAssembly.Module | undefined = undefined;
|
||||||
// 3:1 decimation from AudioContext sample rate to 16 kHz
|
// 3:1 decimation from AudioContext sample rate to 16 kHz
|
||||||
private readonly decRatio = Math.max(1, Math.round(sampleRate / 16000));
|
private readonly decRatio = Math.max(1, Math.round(sampleRate / 16000));
|
||||||
private decPhase = 0;
|
private decPhase = 0;
|
||||||
private decAcc = 0;
|
private decAcc = 0;
|
||||||
|
// Buffer sized for max hop (256); vadHopSize tracks how many samples to collect
|
||||||
private readonly vadHopBuf = new Int16Array(256);
|
private readonly vadHopBuf = new Int16Array(256);
|
||||||
|
private vadHopSize = 256; // standard: 256 (16 ms), aggressive: 160 (10 ms)
|
||||||
private vadHopCount = 0;
|
private vadHopCount = 0;
|
||||||
|
|
||||||
private logCounter = 0;
|
private logCounter = 0;
|
||||||
@@ -229,75 +203,95 @@ class NoiseGateProcessor extends AudioWorkletProcessor {
|
|||||||
super(options);
|
super(options);
|
||||||
|
|
||||||
// Try to instantiate TEN-VAD from the pre-compiled module passed by the main thread
|
// Try to instantiate TEN-VAD from the pre-compiled module passed by the main thread
|
||||||
const tenVadModule = options?.processorOptions?.tenVadModule as
|
this.tenVadModule = options?.processorOptions?.tenVadModule as
|
||||||
| WebAssembly.Module
|
| WebAssembly.Module
|
||||||
| undefined;
|
| undefined;
|
||||||
if (tenVadModule) {
|
if (this.tenVadModule) {
|
||||||
try {
|
try {
|
||||||
// hopSize = 256 samples @ 16 kHz = 16 ms; threshold = 0.5 (overridden via params)
|
// Default: standard mode — 256 samples @ 16 kHz = 16 ms
|
||||||
this.tenVadRuntime = new TenVADRuntime(tenVadModule, 256, 0.5);
|
this.tenVadRuntime = new TenVADRuntime(this.tenVadModule, 256, 0.5);
|
||||||
this.port.postMessage({
|
this.port.postMessage({
|
||||||
type: "log",
|
type: "log",
|
||||||
msg: "[NoiseGate worklet] TEN-VAD runtime initialized, decRatio=" + this.decRatio,
|
msg: "[TenVad worklet] TEN-VAD runtime initialized, decRatio=" + this.decRatio,
|
||||||
});
|
});
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
this.port.postMessage({
|
this.port.postMessage({
|
||||||
type: "log",
|
type: "log",
|
||||||
msg: "[NoiseGate worklet] TEN-VAD init failed: " + String(e),
|
msg: "[TenVad worklet] TEN-VAD init failed: " + String(e),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
this.port.onmessage = (
|
this.port.onmessage = (
|
||||||
e: MessageEvent<NoiseGateParams | VADGateMessage>,
|
e: MessageEvent<TenVadParams | VADGateMessage>,
|
||||||
): void => {
|
): void => {
|
||||||
if ((e.data as VADGateMessage).type === "vad-gate") {
|
if ((e.data as VADGateMessage).type === "vad-gate") {
|
||||||
this.vadGateOpen = (e.data as VADGateMessage).open;
|
this.vadGateOpen = (e.data as VADGateMessage).open;
|
||||||
} else {
|
} else {
|
||||||
this.updateParams(e.data as NoiseGateParams);
|
this.updateParams(e.data as TenVadParams);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
this.updateParams({
|
this.updateParams({
|
||||||
noiseGateActive: true,
|
|
||||||
threshold: -60,
|
|
||||||
attackMs: 25,
|
|
||||||
holdMs: 200,
|
|
||||||
releaseMs: 150,
|
|
||||||
transientEnabled: false,
|
|
||||||
transientThresholdDb: 15,
|
|
||||||
transientReleaseMs: 80,
|
|
||||||
vadEnabled: false,
|
vadEnabled: false,
|
||||||
vadPositiveThreshold: 0.5,
|
vadPositiveThreshold: 0.5,
|
||||||
vadNegativeThreshold: 0.3,
|
vadNegativeThreshold: 0.3,
|
||||||
|
vadMode: "standard",
|
||||||
|
holdMs: 0,
|
||||||
});
|
});
|
||||||
|
|
||||||
this.port.postMessage({
|
this.port.postMessage({
|
||||||
type: "log",
|
type: "log",
|
||||||
msg: "[NoiseGate worklet] constructor called, sampleRate=" + sampleRate,
|
msg: "[TenVad worklet] constructor called, sampleRate=" + sampleRate,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private updateParams(p: NoiseGateParams): void {
|
private updateParams(p: TenVadParams): void {
|
||||||
this.noiseGateActive = p.noiseGateActive ?? true;
|
|
||||||
this.threshold = dbToLinear(p.threshold);
|
|
||||||
this.attackRate = 1.0 / ((p.attackMs / 1000) * sampleRate);
|
|
||||||
this.releaseRate = 1.0 / ((p.releaseMs / 1000) * sampleRate);
|
|
||||||
this.holdTime = p.holdMs / 1000;
|
|
||||||
this.transientEnabled = p.transientEnabled;
|
|
||||||
this.transientRatio = dbToLinear(p.transientThresholdDb);
|
|
||||||
this.transientReleaseRate = 1.0 / ((p.transientReleaseMs / 1000) * sampleRate);
|
|
||||||
this.vadEnabled = p.vadEnabled ?? false;
|
this.vadEnabled = p.vadEnabled ?? false;
|
||||||
this.vadPositiveThreshold = p.vadPositiveThreshold ?? 0.5;
|
this.vadPositiveThreshold = p.vadPositiveThreshold ?? 0.5;
|
||||||
this.vadNegativeThreshold = p.vadNegativeThreshold ?? 0.3;
|
this.vadNegativeThreshold = p.vadNegativeThreshold ?? 0.3;
|
||||||
// When VAD is disabled, open the gate immediately
|
this.holdMs = p.holdMs ?? 0;
|
||||||
|
|
||||||
|
const newMode = p.vadMode ?? "standard";
|
||||||
|
if (newMode === "aggressive") {
|
||||||
|
this.vadOpenRampRate = 1.0 / (0.001 * sampleRate); // 1 ms
|
||||||
|
this.vadCloseRampRate = 1.0 / (0.005 * sampleRate); // 5 ms
|
||||||
|
} else if (newMode === "loose") {
|
||||||
|
this.vadOpenRampRate = 1.0 / (0.012 * sampleRate); // 12 ms
|
||||||
|
this.vadCloseRampRate = 1.0 / (0.032 * sampleRate); // 32 ms
|
||||||
|
} else {
|
||||||
|
this.vadOpenRampRate = 1.0 / (0.005 * sampleRate); // 5 ms
|
||||||
|
this.vadCloseRampRate = 1.0 / (0.02 * sampleRate); // 20 ms
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hop size: aggressive=160 (10 ms @ 16 kHz), others=256 (16 ms)
|
||||||
|
const newHopSize = newMode === "aggressive" ? 160 : 256;
|
||||||
|
if (newHopSize !== this.vadHopSize && this.tenVadModule) {
|
||||||
|
this.tenVadRuntime?.destroy();
|
||||||
|
this.tenVadRuntime = null;
|
||||||
|
this.vadHopCount = 0;
|
||||||
|
try {
|
||||||
|
this.tenVadRuntime = new TenVADRuntime(this.tenVadModule, newHopSize, 0.5);
|
||||||
|
} catch (e) {
|
||||||
|
this.port.postMessage({ type: "log", msg: "[TenVad worklet] TEN-VAD recreate failed: " + String(e) });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
this.vadHopSize = newHopSize;
|
||||||
|
|
||||||
|
// Recompute hold in hops: ceil((holdMs / 1000) * 16000 / vadHopSize)
|
||||||
|
this.vadHoldHops = this.holdMs > 0
|
||||||
|
? Math.ceil((this.holdMs / 1000) * 16000 / this.vadHopSize)
|
||||||
|
: 0;
|
||||||
|
this.vadHoldCounter = 0;
|
||||||
|
|
||||||
if (!this.vadEnabled) this.vadGateOpen = true;
|
if (!this.vadEnabled) this.vadGateOpen = true;
|
||||||
this.port.postMessage({
|
this.port.postMessage({
|
||||||
type: "log",
|
type: "log",
|
||||||
msg: "[NoiseGate worklet] params updated: threshold=" + p.threshold
|
msg: "[TenVad worklet] params updated: vadEnabled=" + p.vadEnabled
|
||||||
+ " vadEnabled=" + p.vadEnabled
|
|
||||||
+ " vadPos=" + p.vadPositiveThreshold
|
+ " vadPos=" + p.vadPositiveThreshold
|
||||||
+ " vadNeg=" + p.vadNegativeThreshold,
|
+ " vadNeg=" + p.vadNegativeThreshold
|
||||||
|
+ " vadMode=" + newMode
|
||||||
|
+ " holdMs=" + this.holdMs,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -306,63 +300,9 @@ class NoiseGateProcessor extends AudioWorkletProcessor {
|
|||||||
const output = outputs[0];
|
const output = outputs[0];
|
||||||
if (!input || input.length === 0) return true;
|
if (!input || input.length === 0) return true;
|
||||||
|
|
||||||
const channels = input.length;
|
|
||||||
const blockSize = input[0]?.length ?? 128;
|
const blockSize = input[0]?.length ?? 128;
|
||||||
const samplePeriod = 1.0 / sampleRate;
|
|
||||||
|
|
||||||
for (let i = 0; i < blockSize; i++) {
|
for (let i = 0; i < blockSize; i++) {
|
||||||
// Peak detection across all channels
|
|
||||||
let curLevel = Math.abs(input[0]?.[i] ?? 0);
|
|
||||||
for (let j = 1; j < channels; j++) {
|
|
||||||
curLevel = Math.max(curLevel, Math.abs(input[j]?.[i] ?? 0));
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- Transient suppressor ---
|
|
||||||
let transientGain = 1.0;
|
|
||||||
if (this.transientEnabled) {
|
|
||||||
this.slowRms = Math.sqrt(
|
|
||||||
this.rmsCoeff * this.slowRms * this.slowRms +
|
|
||||||
(1.0 - this.rmsCoeff) * curLevel * curLevel,
|
|
||||||
);
|
|
||||||
const background = Math.max(this.slowRms, 1e-6);
|
|
||||||
if (curLevel > background * this.transientRatio) {
|
|
||||||
this.transientAttenuation = 0.0;
|
|
||||||
} else {
|
|
||||||
this.transientAttenuation = Math.min(
|
|
||||||
1.0,
|
|
||||||
this.transientAttenuation + this.transientReleaseRate,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
transientGain = this.transientAttenuation;
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- Noise gate ---
|
|
||||||
if (this.noiseGateActive) {
|
|
||||||
if (curLevel > this.threshold && !this.isOpen) {
|
|
||||||
this.isOpen = true;
|
|
||||||
}
|
|
||||||
if (curLevel <= this.threshold && this.isOpen) {
|
|
||||||
this.heldTime = 0;
|
|
||||||
this.isOpen = false;
|
|
||||||
}
|
|
||||||
if (this.isOpen) {
|
|
||||||
this.gateAttenuation = Math.min(
|
|
||||||
1.0,
|
|
||||||
this.gateAttenuation + this.attackRate,
|
|
||||||
);
|
|
||||||
} else {
|
|
||||||
this.heldTime += samplePeriod;
|
|
||||||
if (this.heldTime > this.holdTime) {
|
|
||||||
this.gateAttenuation = Math.max(
|
|
||||||
0.0,
|
|
||||||
this.gateAttenuation - this.releaseRate,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
this.gateAttenuation = 1.0;
|
|
||||||
}
|
|
||||||
|
|
||||||
// --- TEN-VAD in-worklet processing ---
|
// --- TEN-VAD in-worklet processing ---
|
||||||
// Accumulate raw mono samples with decRatio:1 decimation (48 kHz → 16 kHz).
|
// Accumulate raw mono samples with decRatio:1 decimation (48 kHz → 16 kHz).
|
||||||
// Every 256 output samples (16 ms) run the WASM VAD and update vadGateOpen.
|
// Every 256 output samples (16 ms) run the WASM VAD and update vadGateOpen.
|
||||||
@@ -382,33 +322,49 @@ class NoiseGateProcessor extends AudioWorkletProcessor {
|
|||||||
: (avg * 32767 + 0.5) | 0;
|
: (avg * 32767 + 0.5) | 0;
|
||||||
this.vadHopBuf[this.vadHopCount++] = s16;
|
this.vadHopBuf[this.vadHopCount++] = s16;
|
||||||
|
|
||||||
if (this.vadHopCount >= 256) {
|
if (this.vadHopCount >= this.vadHopSize) {
|
||||||
this.vadHopCount = 0;
|
this.vadHopCount = 0;
|
||||||
const prob = this.tenVadRuntime.process(this.vadHopBuf);
|
const prob = this.tenVadRuntime.process(this.vadHopBuf);
|
||||||
if (!this.vadGateOpen && prob >= this.vadPositiveThreshold) {
|
if (prob >= this.vadPositiveThreshold) {
|
||||||
|
// Speech detected — open gate, reset hold counter
|
||||||
this.vadGateOpen = true;
|
this.vadGateOpen = true;
|
||||||
} else if (this.vadGateOpen && prob < this.vadNegativeThreshold) {
|
this.vadHoldCounter = 0;
|
||||||
this.vadGateOpen = false;
|
} else if (prob < this.vadNegativeThreshold) {
|
||||||
|
if (this.vadGateOpen) {
|
||||||
|
if (this.vadHoldHops === 0) {
|
||||||
|
this.vadGateOpen = false;
|
||||||
|
} else {
|
||||||
|
this.vadHoldCounter++;
|
||||||
|
if (this.vadHoldCounter >= this.vadHoldHops) {
|
||||||
|
this.vadGateOpen = false;
|
||||||
|
this.vadHoldCounter = 0;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
// Ambiguous zone — reset hold counter so hold only fires on sustained silence
|
||||||
|
this.vadHoldCounter = 0;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Ramp VAD attenuation toward target to avoid clicks
|
// Asymmetric ramp: fast open (5 ms) to minimise speech onset masking,
|
||||||
|
// slow close (20 ms) to de-click on silence transitions.
|
||||||
const vadTarget = this.vadGateOpen ? 1.0 : 0.0;
|
const vadTarget = this.vadGateOpen ? 1.0 : 0.0;
|
||||||
if (this.vadAttenuation < vadTarget) {
|
if (this.vadAttenuation < vadTarget) {
|
||||||
this.vadAttenuation = Math.min(
|
this.vadAttenuation = Math.min(
|
||||||
vadTarget,
|
vadTarget,
|
||||||
this.vadAttenuation + this.vadRampRate,
|
this.vadAttenuation + this.vadOpenRampRate,
|
||||||
);
|
);
|
||||||
} else if (this.vadAttenuation > vadTarget) {
|
} else if (this.vadAttenuation > vadTarget) {
|
||||||
this.vadAttenuation = Math.max(
|
this.vadAttenuation = Math.max(
|
||||||
vadTarget,
|
vadTarget,
|
||||||
this.vadAttenuation - this.vadRampRate,
|
this.vadAttenuation - this.vadCloseRampRate,
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
const gain = this.gateAttenuation * transientGain * this.vadAttenuation;
|
const gain = this.vadAttenuation;
|
||||||
|
|
||||||
for (let c = 0; c < output.length; c++) {
|
for (let c = 0; c < output.length; c++) {
|
||||||
const inCh = input[c] ?? input[0];
|
const inCh = input[c] ?? input[0];
|
||||||
@@ -423,10 +379,7 @@ class NoiseGateProcessor extends AudioWorkletProcessor {
|
|||||||
if (this.logCounter % 375 === 0) {
|
if (this.logCounter % 375 === 0) {
|
||||||
this.port.postMessage({
|
this.port.postMessage({
|
||||||
type: "log",
|
type: "log",
|
||||||
msg: "[NoiseGate worklet] gateOpen=" + this.isOpen
|
msg: "[TenVad worklet] vadOpen=" + this.vadGateOpen
|
||||||
+ " gateAtten=" + this.gateAttenuation.toFixed(3)
|
|
||||||
+ " transientAtten=" + this.transientAttenuation.toFixed(3)
|
|
||||||
+ " vadOpen=" + this.vadGateOpen
|
|
||||||
+ " vadAtten=" + this.vadAttenuation.toFixed(3),
|
+ " vadAtten=" + this.vadAttenuation.toFixed(3),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -435,4 +388,4 @@ class NoiseGateProcessor extends AudioWorkletProcessor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
registerProcessor("noise-gate-processor", NoiseGateProcessor);
|
registerProcessor("ten-vad-processor", TenVadProcessor);
|
||||||
@@ -7,22 +7,19 @@ Please see LICENSE in the repository root for full details.
|
|||||||
|
|
||||||
import { type Track } from "livekit-client";
|
import { type Track } from "livekit-client";
|
||||||
import { logger } from "matrix-js-sdk/lib/logger";
|
import { logger } from "matrix-js-sdk/lib/logger";
|
||||||
|
// ?worker&url tells Vite to compile the TypeScript worklet and return its URL.
|
||||||
|
// Without this, Vite copies the .ts file verbatim and the browser rejects it.
|
||||||
|
import compiledWorkletUrl from "./TenVadProcessor.worklet.ts?worker&url";
|
||||||
|
|
||||||
const log = logger.getChild("[NoiseGateTransformer]");
|
const log = logger.getChild("[TenVadTransformer]");
|
||||||
|
|
||||||
export interface NoiseGateParams {
|
export interface TenVadParams {
|
||||||
noiseGateActive: boolean;
|
|
||||||
threshold: number; // dBFS — gate opens above this, closes below it
|
|
||||||
attackMs: number;
|
|
||||||
holdMs: number;
|
|
||||||
releaseMs: number;
|
|
||||||
transientEnabled: boolean;
|
|
||||||
transientThresholdDb: number; // dB above background RMS that triggers suppression
|
|
||||||
transientReleaseMs: number; // ms for suppression to fade after transient ends
|
|
||||||
// TEN-VAD params — processed entirely inside the AudioWorklet
|
// TEN-VAD params — processed entirely inside the AudioWorklet
|
||||||
vadEnabled: boolean;
|
vadEnabled: boolean;
|
||||||
vadPositiveThreshold: number; // open gate when isSpeech prob >= this (0–1)
|
vadPositiveThreshold: number; // open gate when prob >= this (0–1)
|
||||||
vadNegativeThreshold: number; // close gate when isSpeech prob < this (0–1)
|
vadNegativeThreshold: number; // close gate when prob < this (0–1); computed by Publisher
|
||||||
|
vadMode: "standard" | "aggressive" | "loose";
|
||||||
|
holdMs: number; // hold time before closing gate (ms); 0 = no hold
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
@@ -68,8 +65,7 @@ function getTenVADModule(): Promise<WebAssembly.Module> {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* LiveKit audio track processor that applies a noise gate, optional transient
|
* LiveKit audio track processor that applies TEN-VAD via AudioWorklet.
|
||||||
* suppressor, and optional TEN-VAD gate via AudioWorklet.
|
|
||||||
*
|
*
|
||||||
* The TEN-VAD WASM module is fetched once, compiled, and passed to the worklet
|
* The TEN-VAD WASM module is fetched once, compiled, and passed to the worklet
|
||||||
* via processorOptions so it runs synchronously inside the audio thread —
|
* via processorOptions so it runs synchronously inside the audio thread —
|
||||||
@@ -78,16 +74,16 @@ function getTenVADModule(): Promise<WebAssembly.Module> {
|
|||||||
* Audio graph: sourceNode → workletNode → destinationNode
|
* Audio graph: sourceNode → workletNode → destinationNode
|
||||||
* processedTrack is destinationNode.stream.getAudioTracks()[0]
|
* processedTrack is destinationNode.stream.getAudioTracks()[0]
|
||||||
*/
|
*/
|
||||||
export class NoiseGateTransformer implements AudioTrackProcessor {
|
export class TenVadTransformer implements AudioTrackProcessor {
|
||||||
public readonly name = "noise-gate";
|
public readonly name = "ten-vad";
|
||||||
public processedTrack?: MediaStreamTrack;
|
public processedTrack?: MediaStreamTrack;
|
||||||
|
|
||||||
private workletNode?: AudioWorkletNode;
|
private workletNode?: AudioWorkletNode;
|
||||||
private sourceNode?: MediaStreamAudioSourceNode;
|
private sourceNode?: MediaStreamAudioSourceNode;
|
||||||
private destinationNode?: MediaStreamAudioDestinationNode;
|
private destinationNode?: MediaStreamAudioDestinationNode;
|
||||||
private params: NoiseGateParams;
|
private params: TenVadParams;
|
||||||
|
|
||||||
public constructor(params: NoiseGateParams) {
|
public constructor(params: TenVadParams) {
|
||||||
this.params = { ...params };
|
this.params = { ...params };
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -105,17 +101,13 @@ export class NoiseGateTransformer implements AudioTrackProcessor {
|
|||||||
log.warn("TEN-VAD WASM module unavailable — VAD disabled:", e);
|
log.warn("TEN-VAD WASM module unavailable — VAD disabled:", e);
|
||||||
}
|
}
|
||||||
|
|
||||||
const workletUrl = new URL(
|
log.info("loading worklet from:", compiledWorkletUrl);
|
||||||
"./NoiseGateProcessor.worklet.ts",
|
await audioContext.audioWorklet.addModule(compiledWorkletUrl);
|
||||||
import.meta.url,
|
|
||||||
);
|
|
||||||
log.info("loading worklet from:", workletUrl.href);
|
|
||||||
await audioContext.audioWorklet.addModule(workletUrl);
|
|
||||||
log.info("worklet module loaded");
|
log.info("worklet module loaded");
|
||||||
|
|
||||||
this.workletNode = new AudioWorkletNode(
|
this.workletNode = new AudioWorkletNode(
|
||||||
audioContext,
|
audioContext,
|
||||||
"noise-gate-processor",
|
"ten-vad-processor",
|
||||||
{
|
{
|
||||||
processorOptions: {
|
processorOptions: {
|
||||||
tenVadModule,
|
tenVadModule,
|
||||||
@@ -158,7 +150,7 @@ export class NoiseGateTransformer implements AudioTrackProcessor {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** Push updated gate/VAD parameters to the running worklet. */
|
/** Push updated gate/VAD parameters to the running worklet. */
|
||||||
public updateParams(params: NoiseGateParams): void {
|
public updateParams(params: TenVadParams): void {
|
||||||
this.params = { ...params };
|
this.params = { ...params };
|
||||||
this.sendParams();
|
this.sendParams();
|
||||||
}
|
}
|
||||||
@@ -1,87 +0,0 @@
|
|||||||
/*
|
|
||||||
Copyright 2026 New Vector Ltd.
|
|
||||||
|
|
||||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
|
||||||
Please see LICENSE in the repository root for full details.
|
|
||||||
*/
|
|
||||||
|
|
||||||
.wrapper {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
position: relative;
|
|
||||||
width: 100%;
|
|
||||||
}
|
|
||||||
|
|
||||||
.slider {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
position: relative;
|
|
||||||
width: 100%;
|
|
||||||
}
|
|
||||||
|
|
||||||
.track {
|
|
||||||
flex-grow: 1;
|
|
||||||
border-radius: var(--cpd-radius-pill-effect);
|
|
||||||
height: var(--cpd-space-4x);
|
|
||||||
outline: var(--cpd-border-width-1) solid var(--cpd-color-border-interactive-primary);
|
|
||||||
outline-offset: calc(-1 * var(--cpd-border-width-1));
|
|
||||||
cursor: pointer;
|
|
||||||
position: relative;
|
|
||||||
overflow: hidden;
|
|
||||||
/* Base background */
|
|
||||||
background: var(--cpd-color-bg-subtle-primary);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Live mic level fill — driven by --mic-level CSS variable */
|
|
||||||
.track::before {
|
|
||||||
content: "";
|
|
||||||
position: absolute;
|
|
||||||
inset: 0;
|
|
||||||
width: var(--mic-level, 0%);
|
|
||||||
background: var(--cpd-color-gray-600, #808080);
|
|
||||||
opacity: 0.55;
|
|
||||||
border-radius: var(--cpd-radius-pill-effect);
|
|
||||||
transition: width 40ms linear;
|
|
||||||
pointer-events: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Green when mic level is at or above the threshold */
|
|
||||||
.track[data-active="true"]::before {
|
|
||||||
background: var(--cpd-color-green-900, #1a7f4b);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Threshold marker — driven by --threshold-pct CSS variable */
|
|
||||||
.track::after {
|
|
||||||
content: "";
|
|
||||||
position: absolute;
|
|
||||||
top: 0;
|
|
||||||
bottom: 0;
|
|
||||||
left: var(--threshold-pct, 50%);
|
|
||||||
width: 2px;
|
|
||||||
background: var(--cpd-color-text-primary);
|
|
||||||
opacity: 0.6;
|
|
||||||
pointer-events: none;
|
|
||||||
transform: translateX(-50%);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Hide the Radix Range highlight — we use our own visuals */
|
|
||||||
.range {
|
|
||||||
display: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.handle {
|
|
||||||
display: block;
|
|
||||||
block-size: var(--cpd-space-5x);
|
|
||||||
inline-size: var(--cpd-space-5x);
|
|
||||||
border-radius: var(--cpd-radius-pill-effect);
|
|
||||||
background: var(--cpd-color-bg-action-primary-rest);
|
|
||||||
box-shadow: 0 0 0 2px var(--cpd-color-bg-canvas-default);
|
|
||||||
cursor: pointer;
|
|
||||||
transition: background-color ease 0.15s;
|
|
||||||
z-index: 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
.handle:focus-visible {
|
|
||||||
outline: 2px solid var(--cpd-color-border-focused);
|
|
||||||
outline-offset: 2px;
|
|
||||||
}
|
|
||||||
@@ -1,139 +0,0 @@
|
|||||||
/*
|
|
||||||
Copyright 2026 New Vector Ltd.
|
|
||||||
|
|
||||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
|
||||||
Please see LICENSE in the repository root for full details.
|
|
||||||
*/
|
|
||||||
|
|
||||||
import { Root, Track, Range, Thumb } from "@radix-ui/react-slider";
|
|
||||||
import { type FC, useCallback, useEffect, useRef } from "react";
|
|
||||||
|
|
||||||
import styles from "./NoiseLevelSlider.module.css";
|
|
||||||
|
|
||||||
interface Props {
|
|
||||||
label: string;
|
|
||||||
value: number;
|
|
||||||
onValueChange: (value: number) => void;
|
|
||||||
onValueCommit?: (value: number) => void;
|
|
||||||
min: number;
|
|
||||||
max: number;
|
|
||||||
step: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Threshold slider that shows live microphone input level as a background fill,
|
|
||||||
* similar to Discord's input sensitivity control.
|
|
||||||
*
|
|
||||||
* The green fill represents your current mic volume in real time.
|
|
||||||
* Drag the handle to set the gate threshold — audio below it will be silenced.
|
|
||||||
*/
|
|
||||||
export const NoiseLevelSlider: FC<Props> = ({
|
|
||||||
label,
|
|
||||||
value,
|
|
||||||
onValueChange: onValueChangeProp,
|
|
||||||
onValueCommit: onValueCommitProp,
|
|
||||||
min,
|
|
||||||
max,
|
|
||||||
step,
|
|
||||||
}) => {
|
|
||||||
const trackRef = useRef<HTMLSpanElement>(null);
|
|
||||||
const animFrameRef = useRef<number>(0);
|
|
||||||
const analyserRef = useRef<AnalyserNode | null>(null);
|
|
||||||
const streamRef = useRef<MediaStream | null>(null);
|
|
||||||
const dataRef = useRef<Float32Array<ArrayBuffer> | null>(null);
|
|
||||||
const thresholdPctRef = useRef<number>(0);
|
|
||||||
|
|
||||||
// Start mic monitoring via AnalyserNode
|
|
||||||
useEffect(() => {
|
|
||||||
let ctx: AudioContext | null = null;
|
|
||||||
|
|
||||||
navigator.mediaDevices
|
|
||||||
.getUserMedia({ audio: true, video: false })
|
|
||||||
.then((stream) => {
|
|
||||||
streamRef.current = stream;
|
|
||||||
ctx = new AudioContext();
|
|
||||||
const source = ctx.createMediaStreamSource(stream);
|
|
||||||
const analyser = ctx.createAnalyser();
|
|
||||||
analyser.fftSize = 1024;
|
|
||||||
analyser.smoothingTimeConstant = 0.6;
|
|
||||||
source.connect(analyser);
|
|
||||||
analyserRef.current = analyser;
|
|
||||||
dataRef.current = new Float32Array(analyser.fftSize) as Float32Array<ArrayBuffer>;
|
|
||||||
})
|
|
||||||
.catch(() => {
|
|
||||||
// Mic not available — live level stays at 0
|
|
||||||
});
|
|
||||||
|
|
||||||
return (): void => {
|
|
||||||
cancelAnimationFrame(animFrameRef.current);
|
|
||||||
streamRef.current?.getTracks().forEach((t) => t.stop());
|
|
||||||
streamRef.current = null;
|
|
||||||
analyserRef.current = null;
|
|
||||||
void ctx?.close();
|
|
||||||
};
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
// rAF loop — reads RMS level and updates CSS variable on the track element
|
|
||||||
useEffect(() => {
|
|
||||||
const tick = (): void => {
|
|
||||||
animFrameRef.current = requestAnimationFrame(tick);
|
|
||||||
|
|
||||||
const analyser = analyserRef.current;
|
|
||||||
const data = dataRef.current;
|
|
||||||
const track = trackRef.current;
|
|
||||||
if (!analyser || !data || !track) return;
|
|
||||||
|
|
||||||
analyser.getFloatTimeDomainData(data);
|
|
||||||
|
|
||||||
// Peak detection — matches the gate's own measurement method
|
|
||||||
let peak = 0;
|
|
||||||
for (const s of data) peak = Math.max(peak, Math.abs(s));
|
|
||||||
const dbfs = peak > 0 ? 20 * Math.log10(peak) : -Infinity;
|
|
||||||
|
|
||||||
// Map dBFS value to slider percentage (same scale as min/max)
|
|
||||||
const clampedDb = Math.max(min, Math.min(max, dbfs));
|
|
||||||
const levelPct = ((clampedDb - min) / (max - min)) * 100;
|
|
||||||
|
|
||||||
track.style.setProperty("--mic-level", `${levelPct.toFixed(1)}%`);
|
|
||||||
track.dataset.active = String(levelPct >= thresholdPctRef.current);
|
|
||||||
};
|
|
||||||
|
|
||||||
animFrameRef.current = requestAnimationFrame(tick);
|
|
||||||
return (): void => cancelAnimationFrame(animFrameRef.current);
|
|
||||||
}, [min, max]);
|
|
||||||
|
|
||||||
// Keep threshold marker in sync with slider value
|
|
||||||
useEffect(() => {
|
|
||||||
const track = trackRef.current;
|
|
||||||
if (!track) return;
|
|
||||||
const thresholdPct = ((value - min) / (max - min)) * 100;
|
|
||||||
thresholdPctRef.current = thresholdPct;
|
|
||||||
track.style.setProperty("--threshold-pct", `${thresholdPct.toFixed(1)}%`);
|
|
||||||
}, [value, min, max]);
|
|
||||||
|
|
||||||
const onValueChange = useCallback(
|
|
||||||
([v]: number[]) => onValueChangeProp(v),
|
|
||||||
[onValueChangeProp],
|
|
||||||
);
|
|
||||||
const onValueCommit = useCallback(
|
|
||||||
([v]: number[]) => onValueCommitProp?.(v),
|
|
||||||
[onValueCommitProp],
|
|
||||||
);
|
|
||||||
|
|
||||||
return (
|
|
||||||
<Root
|
|
||||||
className={styles.slider}
|
|
||||||
value={[value]}
|
|
||||||
onValueChange={onValueChange}
|
|
||||||
onValueCommit={onValueCommit}
|
|
||||||
min={min}
|
|
||||||
max={max}
|
|
||||||
step={step}
|
|
||||||
>
|
|
||||||
<Track className={styles.track} ref={trackRef}>
|
|
||||||
<Range className={styles.range} />
|
|
||||||
</Track>
|
|
||||||
<Thumb className={styles.handle} aria-label={label} />
|
|
||||||
</Root>
|
|
||||||
);
|
|
||||||
};
|
|
||||||
@@ -35,37 +35,30 @@ Please see LICENSE in the repository root for full details.
|
|||||||
color: var(--cpd-color-text-secondary);
|
color: var(--cpd-color-text-secondary);
|
||||||
}
|
}
|
||||||
|
|
||||||
.noiseGateSection {
|
.vadSection {
|
||||||
margin-block-start: var(--cpd-space-6x);
|
margin-block-start: var(--cpd-space-6x);
|
||||||
}
|
}
|
||||||
|
|
||||||
.noiseGateHeading {
|
.vadHeading {
|
||||||
color: var(--cpd-color-text-secondary);
|
color: var(--cpd-color-text-secondary);
|
||||||
margin-block: var(--cpd-space-3x) 0;
|
margin-block: var(--cpd-space-3x) 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
.thresholdSlider {
|
.vadSeparator {
|
||||||
margin-block-start: calc(-32px + var(--cpd-space-2x));
|
|
||||||
}
|
|
||||||
|
|
||||||
.noiseGateSeparator {
|
|
||||||
margin-block: 6px var(--cpd-space-4x);
|
margin-block: 6px var(--cpd-space-4x);
|
||||||
}
|
}
|
||||||
|
|
||||||
.advancedGate {
|
.vadRampLabel {
|
||||||
margin-top: var(--cpd-space-3x);
|
display: block;
|
||||||
|
margin-block: var(--cpd-space-6x) var(--cpd-space-1x);
|
||||||
}
|
}
|
||||||
|
|
||||||
.advancedGateToggle {
|
.vadRampForm {
|
||||||
all: unset;
|
margin-top: 0;
|
||||||
cursor: pointer;
|
|
||||||
font: var(--cpd-font-body-sm-semibold);
|
|
||||||
color: var(--cpd-color-text-secondary);
|
|
||||||
user-select: none;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
.advancedGateToggle:hover {
|
.vadSpacedSlider {
|
||||||
color: var(--cpd-color-text-primary);
|
margin-block-start: var(--cpd-space-6x);
|
||||||
}
|
}
|
||||||
|
|
||||||
.restoreDefaults {
|
.restoreDefaults {
|
||||||
|
|||||||
@@ -5,10 +5,19 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
|||||||
Please see LICENSE in the repository root for full details.
|
Please see LICENSE in the repository root for full details.
|
||||||
*/
|
*/
|
||||||
|
|
||||||
import { type ChangeEvent, type FC, type ReactNode, useEffect, useState, useCallback } from "react";
|
import { type FC, type ReactNode, useEffect, useId, useState } from "react";
|
||||||
import { useTranslation } from "react-i18next";
|
import { useTranslation } from "react-i18next";
|
||||||
import { type MatrixClient } from "matrix-js-sdk";
|
import { type MatrixClient } from "matrix-js-sdk";
|
||||||
import { Button, Heading, Root as Form, Separator } from "@vector-im/compound-web";
|
import {
|
||||||
|
Button,
|
||||||
|
Heading,
|
||||||
|
HelpMessage,
|
||||||
|
InlineField,
|
||||||
|
Label,
|
||||||
|
RadioControl,
|
||||||
|
Root as Form,
|
||||||
|
Separator,
|
||||||
|
} from "@vector-im/compound-web";
|
||||||
import { type Room as LivekitRoom } from "livekit-client";
|
import { type Room as LivekitRoom } from "livekit-client";
|
||||||
|
|
||||||
import { Modal } from "../Modal";
|
import { Modal } from "../Modal";
|
||||||
@@ -24,21 +33,16 @@ import {
|
|||||||
soundEffectVolume as soundEffectVolumeSetting,
|
soundEffectVolume as soundEffectVolumeSetting,
|
||||||
backgroundBlur as backgroundBlurSetting,
|
backgroundBlur as backgroundBlurSetting,
|
||||||
developerMode,
|
developerMode,
|
||||||
noiseGateEnabled as noiseGateEnabledSetting,
|
|
||||||
noiseGateThreshold as noiseGateThresholdSetting,
|
|
||||||
noiseGateAttack as noiseGateAttackSetting,
|
|
||||||
noiseGateHold as noiseGateHoldSetting,
|
|
||||||
noiseGateRelease as noiseGateReleaseSetting,
|
|
||||||
transientSuppressorEnabled as transientSuppressorEnabledSetting,
|
|
||||||
transientThreshold as transientThresholdSetting,
|
|
||||||
transientRelease as transientReleaseSetting,
|
|
||||||
vadEnabled as vadEnabledSetting,
|
vadEnabled as vadEnabledSetting,
|
||||||
vadPositiveThreshold as vadPositiveThresholdSetting,
|
vadPositiveThreshold as vadPositiveThresholdSetting,
|
||||||
vadNegativeThreshold as vadNegativeThresholdSetting,
|
vadMode as vadModeSetting,
|
||||||
|
vadAdvancedEnabled as vadAdvancedEnabledSetting,
|
||||||
|
vadAdvancedOpenThreshold as vadAdvancedOpenThresholdSetting,
|
||||||
|
vadAdvancedCloseThreshold as vadAdvancedCloseThresholdSetting,
|
||||||
|
vadHoldTime as vadHoldTimeSetting,
|
||||||
} from "./settings";
|
} from "./settings";
|
||||||
import { PreferencesSettingsTab } from "./PreferencesSettingsTab";
|
import { PreferencesSettingsTab } from "./PreferencesSettingsTab";
|
||||||
import { Slider } from "../Slider";
|
import { Slider } from "../Slider";
|
||||||
import { NoiseLevelSlider } from "./NoiseLevelSlider";
|
|
||||||
import { DeviceSelection } from "./DeviceSelection";
|
import { DeviceSelection } from "./DeviceSelection";
|
||||||
import { useTrackProcessor } from "../livekit/TrackProcessorContext";
|
import { useTrackProcessor } from "../livekit/TrackProcessorContext";
|
||||||
import { DeveloperSettingsTab } from "./DeveloperSettingsTab";
|
import { DeveloperSettingsTab } from "./DeveloperSettingsTab";
|
||||||
@@ -119,48 +123,25 @@ export const SettingsModal: FC<Props> = ({
|
|||||||
const [soundVolumeRaw, setSoundVolumeRaw] = useState(soundVolume);
|
const [soundVolumeRaw, setSoundVolumeRaw] = useState(soundVolume);
|
||||||
const [showDeveloperSettingsTab] = useSetting(developerMode);
|
const [showDeveloperSettingsTab] = useSetting(developerMode);
|
||||||
|
|
||||||
// Noise gate settings
|
|
||||||
const [noiseGateEnabled, setNoiseGateEnabled] = useSetting(noiseGateEnabledSetting);
|
|
||||||
const [noiseGateThreshold, setNoiseGateThreshold] = useSetting(noiseGateThresholdSetting);
|
|
||||||
const [noiseGateThresholdRaw, setNoiseGateThresholdRaw] = useState(noiseGateThreshold);
|
|
||||||
const [noiseGateAttack, setNoiseGateAttack] = useSetting(noiseGateAttackSetting);
|
|
||||||
const [noiseGateAttackRaw, setNoiseGateAttackRaw] = useState(noiseGateAttack);
|
|
||||||
const [noiseGateHold, setNoiseGateHold] = useSetting(noiseGateHoldSetting);
|
|
||||||
const [noiseGateHoldRaw, setNoiseGateHoldRaw] = useState(noiseGateHold);
|
|
||||||
const [noiseGateRelease, setNoiseGateRelease] = useSetting(noiseGateReleaseSetting);
|
|
||||||
const [noiseGateReleaseRaw, setNoiseGateReleaseRaw] = useState(noiseGateRelease);
|
|
||||||
|
|
||||||
const [showAdvancedGate, setShowAdvancedGate] = useState(false);
|
|
||||||
|
|
||||||
// Voice activity detection
|
// Voice activity detection
|
||||||
|
const vadStateGroup = useId();
|
||||||
|
const vadModeRadioGroup = useId();
|
||||||
const [vadActive, setVadActive] = useSetting(vadEnabledSetting);
|
const [vadActive, setVadActive] = useSetting(vadEnabledSetting);
|
||||||
const [vadPositiveThreshold, setVadPositiveThreshold] = useSetting(vadPositiveThresholdSetting);
|
const [vadSensitivity, setVadSensitivity] = useSetting(vadPositiveThresholdSetting);
|
||||||
const [vadPositiveThresholdRaw, setVadPositiveThresholdRaw] = useState(vadPositiveThreshold);
|
const [vadSensitivityRaw, setVadSensitivityRaw] = useState(vadSensitivity);
|
||||||
const [vadNegativeThreshold, setVadNegativeThreshold] = useSetting(vadNegativeThresholdSetting);
|
const [vadAdvanced, setVadAdvanced] = useSetting(vadAdvancedEnabledSetting);
|
||||||
const [vadNegativeThresholdRaw, setVadNegativeThresholdRaw] = useState(vadNegativeThreshold);
|
const vadState = !vadActive ? "disabled" : vadAdvanced ? "advanced" : "simple";
|
||||||
|
const setVadState = (s: "disabled" | "simple" | "advanced"): void => {
|
||||||
// Transient suppressor settings
|
setVadActive(s !== "disabled");
|
||||||
const [transientEnabled, setTransientEnabled] = useSetting(transientSuppressorEnabledSetting);
|
setVadAdvanced(s === "advanced");
|
||||||
const [transientThreshold, setTransientThreshold] = useSetting(transientThresholdSetting);
|
};
|
||||||
const [transientThresholdRaw, setTransientThresholdRaw] = useState(transientThreshold);
|
const [vadModeValue, setVadModeValue] = useSetting(vadModeSetting);
|
||||||
const [transientRelease, setTransientRelease] = useSetting(transientReleaseSetting);
|
const [vadAdvOpen, setVadAdvOpen] = useSetting(vadAdvancedOpenThresholdSetting);
|
||||||
const [transientReleaseRaw, setTransientReleaseRaw] = useState(transientRelease);
|
const [vadAdvOpenRaw, setVadAdvOpenRaw] = useState(vadAdvOpen);
|
||||||
|
const [vadAdvClose, setVadAdvClose] = useSetting(vadAdvancedCloseThresholdSetting);
|
||||||
const resetTransientDefaults = useCallback((): void => {
|
const [vadAdvCloseRaw, setVadAdvCloseRaw] = useState(vadAdvClose);
|
||||||
const t = transientThresholdSetting.defaultValue;
|
const [vadHold, setVadHold] = useSetting(vadHoldTimeSetting);
|
||||||
const r = transientReleaseSetting.defaultValue;
|
const [vadHoldRaw, setVadHoldRaw] = useState(vadHold);
|
||||||
setTransientThreshold(t); setTransientThresholdRaw(t);
|
|
||||||
setTransientRelease(r); setTransientReleaseRaw(r);
|
|
||||||
}, [setTransientThreshold, setTransientRelease]);
|
|
||||||
|
|
||||||
const resetGateDefaults = useCallback((): void => {
|
|
||||||
const a = noiseGateAttackSetting.defaultValue;
|
|
||||||
const h = noiseGateHoldSetting.defaultValue;
|
|
||||||
const r = noiseGateReleaseSetting.defaultValue;
|
|
||||||
setNoiseGateAttack(a); setNoiseGateAttackRaw(a);
|
|
||||||
setNoiseGateHold(h); setNoiseGateHoldRaw(h);
|
|
||||||
setNoiseGateRelease(r); setNoiseGateReleaseRaw(r);
|
|
||||||
}, [setNoiseGateAttack, setNoiseGateHold, setNoiseGateRelease]);
|
|
||||||
|
|
||||||
const { available: isRageshakeAvailable } = useSubmitRageshake();
|
const { available: isRageshakeAvailable } = useSubmitRageshake();
|
||||||
|
|
||||||
@@ -220,234 +201,185 @@ export const SettingsModal: FC<Props> = ({
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
</Form>
|
</Form>
|
||||||
<div className={styles.noiseGateSection}>
|
<div className={styles.vadSection}>
|
||||||
<Heading
|
<Heading
|
||||||
type="body"
|
type="body"
|
||||||
weight="semibold"
|
weight="semibold"
|
||||||
size="sm"
|
size="sm"
|
||||||
as="h4"
|
as="h4"
|
||||||
className={styles.noiseGateHeading}
|
className={styles.vadHeading}
|
||||||
>
|
>
|
||||||
Noise Gate
|
Voice Activity Detection
|
||||||
</Heading>
|
</Heading>
|
||||||
<Separator className={styles.noiseGateSeparator} />
|
<Separator className={styles.vadSeparator} />
|
||||||
<FieldRow>
|
<Form>
|
||||||
<InputField
|
<InlineField
|
||||||
id="noiseGateEnabled"
|
name={vadStateGroup}
|
||||||
type="checkbox"
|
control={
|
||||||
label="Enable noise gate"
|
<RadioControl
|
||||||
description="Suppress audio below a configurable threshold."
|
checked={vadState === "disabled"}
|
||||||
checked={noiseGateEnabled}
|
value="disabled"
|
||||||
onChange={(e: ChangeEvent<HTMLInputElement>): void =>
|
onChange={(): void => setVadState("disabled")}
|
||||||
setNoiseGateEnabled(e.target.checked)
|
/>
|
||||||
}
|
}
|
||||||
/>
|
>
|
||||||
</FieldRow>
|
<Label>Disabled</Label>
|
||||||
{noiseGateEnabled && (
|
</InlineField>
|
||||||
<>
|
<InlineField
|
||||||
<div className={`${styles.volumeSlider} ${styles.thresholdSlider}`}>
|
name={vadStateGroup}
|
||||||
<span className={styles.sliderLabel}>Threshold</span>
|
control={
|
||||||
<p>Gate opens above this level, closes below it.</p>
|
<RadioControl
|
||||||
<NoiseLevelSlider
|
checked={vadState === "simple"}
|
||||||
label="Noise gate threshold"
|
value="simple"
|
||||||
value={noiseGateThresholdRaw}
|
onChange={(): void => setVadState("simple")}
|
||||||
onValueChange={setNoiseGateThresholdRaw}
|
/>
|
||||||
onValueCommit={setNoiseGateThreshold}
|
}
|
||||||
min={-100}
|
>
|
||||||
max={0}
|
<Label>Simple</Label>
|
||||||
step={1}
|
</InlineField>
|
||||||
/>
|
<InlineField
|
||||||
</div>
|
name={vadStateGroup}
|
||||||
<div className={styles.advancedGate}>
|
control={
|
||||||
<button
|
<RadioControl
|
||||||
className={styles.advancedGateToggle}
|
checked={vadState === "advanced"}
|
||||||
onClick={(): void => setShowAdvancedGate((v) => !v)}
|
value="advanced"
|
||||||
>
|
onChange={(): void => setVadState("advanced")}
|
||||||
{showAdvancedGate ? "▾" : "▸"} Advanced settings
|
/>
|
||||||
</button>
|
}
|
||||||
{showAdvancedGate && (
|
>
|
||||||
|
<Label>Advanced</Label>
|
||||||
|
</InlineField>
|
||||||
|
</Form>
|
||||||
|
{vadState !== "disabled" && (
|
||||||
|
<>
|
||||||
|
{vadState === "simple" && (
|
||||||
|
<div className={styles.volumeSlider}>
|
||||||
|
<span className={styles.sliderLabel}>
|
||||||
|
Sensitivity: {Math.round(vadSensitivityRaw * 100)}%
|
||||||
|
</span>
|
||||||
|
<p>Higher values require more confident speech detection before opening.</p>
|
||||||
|
<Slider
|
||||||
|
label="VAD sensitivity"
|
||||||
|
value={vadSensitivityRaw}
|
||||||
|
onValueChange={setVadSensitivityRaw}
|
||||||
|
onValueCommit={setVadSensitivity}
|
||||||
|
min={0.1}
|
||||||
|
max={1.0}
|
||||||
|
step={0.05}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{vadState === "advanced" && (
|
||||||
<>
|
<>
|
||||||
<div className={styles.volumeSlider}>
|
<span className={styles.vadRampLabel}>Ramp profiles</span>
|
||||||
<label>Attack: {noiseGateAttackRaw} ms</label>
|
<Form className={styles.vadRampForm}>
|
||||||
<p>How quickly the gate opens when signal exceeds threshold.</p>
|
<InlineField
|
||||||
|
name={vadModeRadioGroup}
|
||||||
|
control={
|
||||||
|
<RadioControl
|
||||||
|
checked={vadModeValue === "loose"}
|
||||||
|
value="loose"
|
||||||
|
onChange={(): void => setVadModeValue("loose")}
|
||||||
|
/>
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<Label>Loose</Label>
|
||||||
|
<HelpMessage>256 samples / 16 ms — 12 ms open / 32 ms close ramp.</HelpMessage>
|
||||||
|
</InlineField>
|
||||||
|
<InlineField
|
||||||
|
name={vadModeRadioGroup}
|
||||||
|
control={
|
||||||
|
<RadioControl
|
||||||
|
checked={vadModeValue === "standard"}
|
||||||
|
value="standard"
|
||||||
|
onChange={(): void => setVadModeValue("standard")}
|
||||||
|
/>
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<Label>Standard</Label>
|
||||||
|
<HelpMessage>256 samples / 16 ms — 5 ms open / 20 ms close ramp.</HelpMessage>
|
||||||
|
</InlineField>
|
||||||
|
<InlineField
|
||||||
|
name={vadModeRadioGroup}
|
||||||
|
control={
|
||||||
|
<RadioControl
|
||||||
|
checked={vadModeValue === "aggressive"}
|
||||||
|
value="aggressive"
|
||||||
|
onChange={(): void => setVadModeValue("aggressive")}
|
||||||
|
/>
|
||||||
|
}
|
||||||
|
>
|
||||||
|
<Label>Aggressive</Label>
|
||||||
|
<HelpMessage>160 samples / 10 ms — 1 ms open / 5 ms close ramp.</HelpMessage>
|
||||||
|
</InlineField>
|
||||||
|
</Form>
|
||||||
|
<div className={`${styles.volumeSlider} ${styles.vadSpacedSlider}`}>
|
||||||
|
<span className={styles.sliderLabel}>
|
||||||
|
Open threshold: {Math.round(vadAdvOpenRaw * 100)}%
|
||||||
|
</span>
|
||||||
|
<p>Minimum confidence required to open the gate.</p>
|
||||||
<Slider
|
<Slider
|
||||||
label="Noise gate attack"
|
label="VAD open threshold"
|
||||||
value={noiseGateAttackRaw}
|
value={vadAdvOpenRaw}
|
||||||
onValueChange={setNoiseGateAttackRaw}
|
onValueChange={setVadAdvOpenRaw}
|
||||||
onValueCommit={setNoiseGateAttack}
|
onValueCommit={setVadAdvOpen}
|
||||||
min={1}
|
min={0.1}
|
||||||
max={100}
|
max={0.95}
|
||||||
step={1}
|
step={0.05}
|
||||||
tooltip={false}
|
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
<div className={styles.volumeSlider}>
|
<div className={styles.volumeSlider}>
|
||||||
<label>Hold: {noiseGateHoldRaw} ms</label>
|
<span className={styles.sliderLabel}>
|
||||||
<p>How long the gate stays open after signal drops below threshold.</p>
|
Close threshold: {Math.round(vadAdvCloseRaw * 100)}%
|
||||||
|
</span>
|
||||||
|
<p>Probability must drop below this to start the hold/close sequence.</p>
|
||||||
<Slider
|
<Slider
|
||||||
label="Noise gate hold"
|
label="VAD close threshold"
|
||||||
value={noiseGateHoldRaw}
|
value={vadAdvCloseRaw}
|
||||||
onValueChange={setNoiseGateHoldRaw}
|
onValueChange={setVadAdvCloseRaw}
|
||||||
onValueCommit={setNoiseGateHold}
|
onValueCommit={setVadAdvClose}
|
||||||
|
min={0.05}
|
||||||
|
max={0.9}
|
||||||
|
step={0.05}
|
||||||
|
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<div className={`${styles.volumeSlider} ${styles.vadSpacedSlider}`}>
|
||||||
|
<span className={styles.sliderLabel}>
|
||||||
|
Hold time: {vadHoldRaw} ms
|
||||||
|
</span>
|
||||||
|
<p>How long to keep the gate open after speech drops below the close threshold.</p>
|
||||||
|
<Slider
|
||||||
|
label="VAD hold time"
|
||||||
|
value={vadHoldRaw}
|
||||||
|
onValueChange={setVadHoldRaw}
|
||||||
|
onValueCommit={setVadHold}
|
||||||
min={0}
|
min={0}
|
||||||
max={500}
|
max={2000}
|
||||||
step={10}
|
step={50}
|
||||||
tooltip={false}
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
<div className={styles.volumeSlider}>
|
|
||||||
<label>Release: {noiseGateReleaseRaw} ms</label>
|
|
||||||
<p>How quickly the gate closes after hold expires.</p>
|
|
||||||
<Slider
|
|
||||||
label="Noise gate release"
|
|
||||||
value={noiseGateReleaseRaw}
|
|
||||||
onValueChange={setNoiseGateReleaseRaw}
|
|
||||||
onValueCommit={setNoiseGateRelease}
|
|
||||||
min={10}
|
|
||||||
max={500}
|
|
||||||
step={10}
|
|
||||||
tooltip={false}
|
|
||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
<div className={styles.restoreDefaults}>
|
<div className={styles.restoreDefaults}>
|
||||||
<Button kind="secondary" size="sm" onClick={resetGateDefaults}>
|
<Button
|
||||||
|
kind="secondary"
|
||||||
|
size="sm"
|
||||||
|
onClick={(): void => {
|
||||||
|
const defOpen = vadAdvancedOpenThresholdSetting.defaultValue;
|
||||||
|
const defClose = vadAdvancedCloseThresholdSetting.defaultValue;
|
||||||
|
const defHold = vadHoldTimeSetting.defaultValue;
|
||||||
|
setVadAdvOpen(defOpen); setVadAdvOpenRaw(defOpen);
|
||||||
|
setVadAdvClose(defClose); setVadAdvCloseRaw(defClose);
|
||||||
|
setVadHold(defHold); setVadHoldRaw(defHold);
|
||||||
|
setVadModeValue("standard");
|
||||||
|
}}
|
||||||
|
>
|
||||||
Restore defaults
|
Restore defaults
|
||||||
</Button>
|
</Button>
|
||||||
</div>
|
</div>
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
</div>
|
|
||||||
</>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
<div className={styles.noiseGateSection}>
|
|
||||||
<Heading
|
|
||||||
type="body"
|
|
||||||
weight="semibold"
|
|
||||||
size="sm"
|
|
||||||
as="h4"
|
|
||||||
className={styles.noiseGateHeading}
|
|
||||||
>
|
|
||||||
Voice Activity Detection
|
|
||||||
</Heading>
|
|
||||||
<Separator className={styles.noiseGateSeparator} />
|
|
||||||
<FieldRow>
|
|
||||||
<InputField
|
|
||||||
id="vadEnabled"
|
|
||||||
type="checkbox"
|
|
||||||
label="Enable voice activity detection"
|
|
||||||
description="Uses TEN-VAD to mute audio when no speech is detected (~16 ms latency)."
|
|
||||||
checked={vadActive}
|
|
||||||
onChange={(e: ChangeEvent<HTMLInputElement>): void =>
|
|
||||||
setVadActive(e.target.checked)
|
|
||||||
}
|
|
||||||
/>
|
|
||||||
</FieldRow>
|
|
||||||
{vadActive && (
|
|
||||||
<>
|
|
||||||
<div className={`${styles.volumeSlider} ${styles.thresholdSlider}`}>
|
|
||||||
<span className={styles.sliderLabel}>Open threshold: {Math.round(vadPositiveThresholdRaw * 100)}%</span>
|
|
||||||
<p>How confident the model must be before opening the gate.</p>
|
|
||||||
<Slider
|
|
||||||
label="VAD open threshold"
|
|
||||||
value={vadPositiveThresholdRaw}
|
|
||||||
onValueChange={setVadPositiveThresholdRaw}
|
|
||||||
onValueCommit={setVadPositiveThreshold}
|
|
||||||
min={0.1}
|
|
||||||
max={0.9}
|
|
||||||
step={0.05}
|
|
||||||
tooltip={false}
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
<div className={styles.volumeSlider}>
|
|
||||||
<span className={styles.sliderLabel}>Close threshold: {Math.round(vadNegativeThresholdRaw * 100)}%</span>
|
|
||||||
<p>How low the probability must drop before closing the gate.</p>
|
|
||||||
<Slider
|
|
||||||
label="VAD close threshold"
|
|
||||||
value={vadNegativeThresholdRaw}
|
|
||||||
onValueChange={setVadNegativeThresholdRaw}
|
|
||||||
onValueCommit={setVadNegativeThreshold}
|
|
||||||
min={0.05}
|
|
||||||
max={0.7}
|
|
||||||
step={0.05}
|
|
||||||
tooltip={false}
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
<div className={styles.restoreDefaults}>
|
|
||||||
<Button
|
|
||||||
kind="secondary"
|
|
||||||
size="sm"
|
|
||||||
onClick={(): void => {
|
|
||||||
const pos = vadPositiveThresholdSetting.defaultValue;
|
|
||||||
const neg = vadNegativeThresholdSetting.defaultValue;
|
|
||||||
setVadPositiveThreshold(pos); setVadPositiveThresholdRaw(pos);
|
|
||||||
setVadNegativeThreshold(neg); setVadNegativeThresholdRaw(neg);
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
Restore defaults
|
|
||||||
</Button>
|
|
||||||
</div>
|
|
||||||
</>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
<div className={styles.noiseGateSection}>
|
|
||||||
<Heading
|
|
||||||
type="body"
|
|
||||||
weight="semibold"
|
|
||||||
size="sm"
|
|
||||||
as="h4"
|
|
||||||
className={styles.noiseGateHeading}
|
|
||||||
>
|
|
||||||
Transient Suppressor
|
|
||||||
</Heading>
|
|
||||||
<Separator className={styles.noiseGateSeparator} />
|
|
||||||
<FieldRow>
|
|
||||||
<InputField
|
|
||||||
id="transientEnabled"
|
|
||||||
type="checkbox"
|
|
||||||
label="Enable transient suppressor"
|
|
||||||
description="Cut sudden loud impacts like desk hits or mic bumps."
|
|
||||||
checked={transientEnabled}
|
|
||||||
onChange={(e: ChangeEvent<HTMLInputElement>): void =>
|
|
||||||
setTransientEnabled(e.target.checked)
|
|
||||||
}
|
|
||||||
/>
|
|
||||||
</FieldRow>
|
|
||||||
{transientEnabled && (
|
|
||||||
<>
|
|
||||||
<div className={`${styles.volumeSlider} ${styles.thresholdSlider}`}>
|
|
||||||
<span className={styles.sliderLabel}>Sensitivity: {transientThresholdRaw} dB above background</span>
|
|
||||||
<p>Lower values catch more impacts; higher values only catch the loudest ones.</p>
|
|
||||||
<Slider
|
|
||||||
label="Transient threshold"
|
|
||||||
value={transientThresholdRaw}
|
|
||||||
onValueChange={setTransientThresholdRaw}
|
|
||||||
onValueCommit={setTransientThreshold}
|
|
||||||
min={8}
|
|
||||||
max={30}
|
|
||||||
step={1}
|
|
||||||
tooltip={false}
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
<div className={styles.volumeSlider}>
|
|
||||||
<span className={styles.sliderLabel}>Release: {transientReleaseRaw} ms</span>
|
|
||||||
<p>How quickly audio returns after suppression.</p>
|
|
||||||
<Slider
|
|
||||||
label="Transient release"
|
|
||||||
value={transientReleaseRaw}
|
|
||||||
onValueChange={setTransientReleaseRaw}
|
|
||||||
onValueCommit={setTransientRelease}
|
|
||||||
min={20}
|
|
||||||
max={200}
|
|
||||||
step={10}
|
|
||||||
tooltip={false}
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
<div className={styles.restoreDefaults}>
|
|
||||||
<Button kind="secondary" size="sm" onClick={resetTransientDefaults}>
|
|
||||||
Restore defaults
|
|
||||||
</Button>
|
|
||||||
</div>
|
|
||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -129,42 +129,16 @@ export const alwaysShowIphoneEarpiece = new Setting<boolean>(
|
|||||||
false,
|
false,
|
||||||
);
|
);
|
||||||
|
|
||||||
export const noiseGateEnabled = new Setting<boolean>(
|
|
||||||
"noise-gate-enabled",
|
|
||||||
false,
|
|
||||||
);
|
|
||||||
// Threshold in dBFS — gate opens above this, closes below it
|
|
||||||
export const noiseGateThreshold = new Setting<number>(
|
|
||||||
"noise-gate-threshold",
|
|
||||||
-60,
|
|
||||||
);
|
|
||||||
// Time in ms for the gate to fully open after signal exceeds threshold
|
|
||||||
export const noiseGateAttack = new Setting<number>("noise-gate-attack", 25);
|
|
||||||
// Time in ms the gate stays open after signal drops below threshold
|
|
||||||
export const noiseGateHold = new Setting<number>("noise-gate-hold", 200);
|
|
||||||
// Time in ms for the gate to fully close after hold expires
|
|
||||||
export const noiseGateRelease = new Setting<number>("noise-gate-release", 150);
|
|
||||||
|
|
||||||
export const vadEnabled = new Setting<boolean>("vad-enabled", false);
|
export const vadEnabled = new Setting<boolean>("vad-enabled", false);
|
||||||
// Probability above which the VAD opens the gate (0–1)
|
// Simple mode: single sensitivity slider (open threshold); close = open - 0.1
|
||||||
export const vadPositiveThreshold = new Setting<number>("vad-positive-threshold", 0.2);
|
export const vadPositiveThreshold = new Setting<number>("vad-positive-threshold", 0.7);
|
||||||
// Probability below which the VAD closes the gate (0–1)
|
// standard: 5ms/20ms aggressive: 1ms/5ms loose: 12ms/32ms
|
||||||
export const vadNegativeThreshold = new Setting<number>("vad-negative-threshold", 0.1);
|
export const vadMode = new Setting<"standard" | "aggressive" | "loose">("vad-mode", "standard");
|
||||||
|
// Advanced settings (override simple mode when enabled)
|
||||||
export const transientSuppressorEnabled = new Setting<boolean>(
|
export const vadAdvancedEnabled = new Setting<boolean>("vad-advanced-enabled", false);
|
||||||
"transient-suppressor-enabled",
|
export const vadAdvancedOpenThreshold = new Setting<number>("vad-advanced-open-threshold", 0.7);
|
||||||
false,
|
export const vadAdvancedCloseThreshold = new Setting<number>("vad-advanced-close-threshold", 0.6);
|
||||||
);
|
export const vadHoldTime = new Setting<number>("vad-hold-time", 300);
|
||||||
// How many dB above the background RMS a peak must be to trigger suppression
|
|
||||||
export const transientThreshold = new Setting<number>(
|
|
||||||
"transient-suppressor-threshold",
|
|
||||||
15,
|
|
||||||
);
|
|
||||||
// Time in ms for suppression to fade after transient ends
|
|
||||||
export const transientRelease = new Setting<number>(
|
|
||||||
"transient-suppressor-release",
|
|
||||||
80,
|
|
||||||
);
|
|
||||||
|
|
||||||
export enum MatrixRTCMode {
|
export enum MatrixRTCMode {
|
||||||
Legacy = "legacy",
|
Legacy = "legacy",
|
||||||
|
|||||||
@@ -33,22 +33,18 @@ import {
|
|||||||
} from "../../../livekit/TrackProcessorContext.tsx";
|
} from "../../../livekit/TrackProcessorContext.tsx";
|
||||||
import { getUrlParams } from "../../../UrlParams.ts";
|
import { getUrlParams } from "../../../UrlParams.ts";
|
||||||
import {
|
import {
|
||||||
noiseGateEnabled,
|
|
||||||
noiseGateThreshold,
|
|
||||||
noiseGateAttack,
|
|
||||||
noiseGateHold,
|
|
||||||
noiseGateRelease,
|
|
||||||
transientSuppressorEnabled,
|
|
||||||
transientThreshold,
|
|
||||||
transientRelease,
|
|
||||||
vadEnabled,
|
vadEnabled,
|
||||||
vadPositiveThreshold,
|
vadPositiveThreshold,
|
||||||
vadNegativeThreshold,
|
vadMode,
|
||||||
|
vadAdvancedEnabled,
|
||||||
|
vadAdvancedOpenThreshold,
|
||||||
|
vadAdvancedCloseThreshold,
|
||||||
|
vadHoldTime,
|
||||||
} from "../../../settings/settings.ts";
|
} from "../../../settings/settings.ts";
|
||||||
import {
|
import {
|
||||||
type NoiseGateParams,
|
type TenVadParams,
|
||||||
NoiseGateTransformer,
|
TenVadTransformer,
|
||||||
} from "../../../livekit/NoiseGateTransformer.ts";
|
} from "../../../livekit/TenVadTransformer.ts";
|
||||||
import { observeTrackReference$ } from "../../observeTrackReference";
|
import { observeTrackReference$ } from "../../observeTrackReference";
|
||||||
import { type Connection } from "../remoteMembers/Connection.ts";
|
import { type Connection } from "../remoteMembers/Connection.ts";
|
||||||
import { ObservableScope } from "../../ObservableScope.ts";
|
import { ObservableScope } from "../../ObservableScope.ts";
|
||||||
@@ -93,7 +89,7 @@ export class Publisher {
|
|||||||
// Setup track processor syncing (blur)
|
// Setup track processor syncing (blur)
|
||||||
this.observeTrackProcessors(this.scope, room, trackerProcessorState$);
|
this.observeTrackProcessors(this.scope, room, trackerProcessorState$);
|
||||||
// Setup noise gate on the local microphone track
|
// Setup noise gate on the local microphone track
|
||||||
this.applyNoiseGate(this.scope, room);
|
this.applyTenVad(this.scope, room);
|
||||||
// Observe media device changes and update LiveKit active devices accordingly
|
// Observe media device changes and update LiveKit active devices accordingly
|
||||||
this.observeMediaDevices(this.scope, devices, controlledAudioDevices);
|
this.observeMediaDevices(this.scope, devices, controlledAudioDevices);
|
||||||
|
|
||||||
@@ -421,7 +417,7 @@ export class Publisher {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private applyNoiseGate(scope: ObservableScope, room: LivekitRoom): void {
|
private applyTenVad(scope: ObservableScope, room: LivekitRoom): void {
|
||||||
// Observe the local microphone track
|
// Observe the local microphone track
|
||||||
const audioTrack$ = scope.behavior(
|
const audioTrack$ = scope.behavior(
|
||||||
observeTrackReference$(
|
observeTrackReference$(
|
||||||
@@ -436,48 +432,55 @@ export class Publisher {
|
|||||||
null,
|
null,
|
||||||
);
|
);
|
||||||
|
|
||||||
let transformer: NoiseGateTransformer | null = null;
|
let transformer: TenVadTransformer | null = null;
|
||||||
let audioCtx: AudioContext | null = null;
|
let audioCtx: AudioContext | null = null;
|
||||||
|
|
||||||
const currentParams = (): NoiseGateParams => ({
|
const currentParams = (): TenVadParams => {
|
||||||
noiseGateActive: noiseGateEnabled.getValue(),
|
const isAdvanced = vadAdvancedEnabled.getValue();
|
||||||
threshold: noiseGateThreshold.getValue(),
|
if (isAdvanced) {
|
||||||
attackMs: noiseGateAttack.getValue(),
|
return {
|
||||||
holdMs: noiseGateHold.getValue(),
|
vadEnabled: vadEnabled.getValue(),
|
||||||
releaseMs: noiseGateRelease.getValue(),
|
vadPositiveThreshold: vadAdvancedOpenThreshold.getValue(),
|
||||||
transientEnabled: transientSuppressorEnabled.getValue(),
|
vadNegativeThreshold: vadAdvancedCloseThreshold.getValue(),
|
||||||
transientThresholdDb: transientThreshold.getValue(),
|
vadMode: vadMode.getValue(),
|
||||||
transientReleaseMs: transientRelease.getValue(),
|
holdMs: vadHoldTime.getValue(),
|
||||||
vadEnabled: vadEnabled.getValue(),
|
};
|
||||||
vadPositiveThreshold: vadPositiveThreshold.getValue(),
|
}
|
||||||
vadNegativeThreshold: vadNegativeThreshold.getValue(),
|
const openT = vadPositiveThreshold.getValue();
|
||||||
});
|
return {
|
||||||
|
vadEnabled: vadEnabled.getValue(),
|
||||||
|
vadPositiveThreshold: openT,
|
||||||
|
vadNegativeThreshold: Math.max(0, openT - 0.1),
|
||||||
|
vadMode: "standard",
|
||||||
|
holdMs: 0,
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
// Attach / detach processor when any processing feature changes or the track changes.
|
// Attach / detach processor when VAD is toggled or the track changes.
|
||||||
combineLatest([audioTrack$, noiseGateEnabled.value$, vadEnabled.value$, transientSuppressorEnabled.value$])
|
combineLatest([audioTrack$, vadEnabled.value$])
|
||||||
.pipe(scope.bind())
|
.pipe(scope.bind())
|
||||||
.subscribe(([audioTrack, ngEnabled, vadActive, transientActive]) => {
|
.subscribe(([audioTrack, vadActive]) => {
|
||||||
if (!audioTrack) return;
|
if (!audioTrack) return;
|
||||||
const shouldAttach = ngEnabled || vadActive || transientActive;
|
const shouldAttach = vadActive;
|
||||||
if (shouldAttach && !audioTrack.getProcessor()) {
|
if (shouldAttach && !audioTrack.getProcessor()) {
|
||||||
const params = currentParams();
|
const params = currentParams();
|
||||||
this.logger.info("[NoiseGate] attaching processor, params:", params);
|
this.logger.info("[TenVad] attaching processor, params:", params);
|
||||||
transformer = new NoiseGateTransformer(params);
|
transformer = new TenVadTransformer(params);
|
||||||
audioCtx = new AudioContext();
|
audioCtx = new AudioContext();
|
||||||
this.logger.info("[NoiseGate] AudioContext state before resume:", audioCtx.state);
|
this.logger.info("[TenVad] AudioContext state before resume:", audioCtx.state);
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
(audioTrack as any).setAudioContext(audioCtx);
|
(audioTrack as any).setAudioContext(audioCtx);
|
||||||
audioCtx.resume().then(async () => {
|
audioCtx.resume().then(async () => {
|
||||||
this.logger.info("[NoiseGate] AudioContext state after resume:", audioCtx?.state);
|
this.logger.info("[TenVad] AudioContext state after resume:", audioCtx?.state);
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
return audioTrack.setProcessor(transformer as any);
|
return audioTrack.setProcessor(transformer as any);
|
||||||
}).then(() => {
|
}).then(() => {
|
||||||
this.logger.info("[NoiseGate] setProcessor resolved");
|
this.logger.info("[TenVad] setProcessor resolved");
|
||||||
}).catch((e: unknown) => {
|
}).catch((e: unknown) => {
|
||||||
this.logger.error("[NoiseGate] setProcessor failed", e);
|
this.logger.error("[TenVad] setProcessor failed", e);
|
||||||
});
|
});
|
||||||
} else if (!shouldAttach && audioTrack.getProcessor()) {
|
} else if (!shouldAttach && audioTrack.getProcessor()) {
|
||||||
this.logger.info("[NoiseGate] removing processor");
|
this.logger.info("[TenVad] removing processor");
|
||||||
void audioTrack.stopProcessor();
|
void audioTrack.stopProcessor();
|
||||||
void audioCtx?.close();
|
void audioCtx?.close();
|
||||||
audioCtx = null;
|
audioCtx = null;
|
||||||
@@ -485,44 +488,29 @@ export class Publisher {
|
|||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
(audioTrack as any).setAudioContext(undefined);
|
(audioTrack as any).setAudioContext(undefined);
|
||||||
} else if (shouldAttach && audioTrack.getProcessor()) {
|
} else if (shouldAttach && audioTrack.getProcessor()) {
|
||||||
// Processor already attached — push updated params (e.g. noiseGateActive toggled)
|
// Processor already attached — push updated params (e.g. vadActive toggled)
|
||||||
transformer?.updateParams(currentParams());
|
transformer?.updateParams(currentParams());
|
||||||
} else {
|
} else {
|
||||||
this.logger.info(
|
this.logger.info(
|
||||||
"[NoiseGate] tick — ngEnabled:", ngEnabled,
|
"[TenVad] tick — vadActive:", vadActive,
|
||||||
"vadActive:", vadActive,
|
|
||||||
"hasProcessor:", !!audioTrack.getProcessor(),
|
"hasProcessor:", !!audioTrack.getProcessor(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
// Push all param changes (noise gate + VAD) to the live worklet.
|
// Push VAD param changes to the live worklet.
|
||||||
combineLatest([
|
combineLatest([
|
||||||
noiseGateEnabled.value$,
|
|
||||||
noiseGateThreshold.value$,
|
|
||||||
noiseGateAttack.value$,
|
|
||||||
noiseGateHold.value$,
|
|
||||||
noiseGateRelease.value$,
|
|
||||||
transientSuppressorEnabled.value$,
|
|
||||||
transientThreshold.value$,
|
|
||||||
transientRelease.value$,
|
|
||||||
vadEnabled.value$,
|
vadEnabled.value$,
|
||||||
vadPositiveThreshold.value$,
|
vadPositiveThreshold.value$,
|
||||||
vadNegativeThreshold.value$,
|
vadMode.value$,
|
||||||
|
vadAdvancedEnabled.value$,
|
||||||
|
vadAdvancedOpenThreshold.value$,
|
||||||
|
vadAdvancedCloseThreshold.value$,
|
||||||
|
vadHoldTime.value$,
|
||||||
])
|
])
|
||||||
.pipe(scope.bind())
|
.pipe(scope.bind())
|
||||||
.subscribe(([
|
.subscribe(() => {
|
||||||
noiseGateActive, threshold, attackMs, holdMs, releaseMs,
|
transformer?.updateParams(currentParams());
|
||||||
transientEnabled, transientThresholdDb, transientReleaseMs,
|
|
||||||
vadActive, vadPos, vadNeg,
|
|
||||||
]) => {
|
|
||||||
transformer?.updateParams({
|
|
||||||
noiseGateActive, threshold, attackMs, holdMs, releaseMs,
|
|
||||||
transientEnabled, transientThresholdDb, transientReleaseMs,
|
|
||||||
vadEnabled: vadActive,
|
|
||||||
vadPositiveThreshold: vadPos,
|
|
||||||
vadNegativeThreshold: vadNeg,
|
|
||||||
});
|
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -542,5 +530,4 @@ export class Publisher {
|
|||||||
);
|
);
|
||||||
trackProcessorSync(scope, track$, trackerProcessorState$);
|
trackProcessorSync(scope, track$, trackerProcessorState$);
|
||||||
}
|
}
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
|
|||||||
|
|
||||||
import {
|
import {
|
||||||
loadEnv,
|
loadEnv,
|
||||||
|
PluginOption,
|
||||||
searchForWorkspaceRoot,
|
searchForWorkspaceRoot,
|
||||||
type ConfigEnv,
|
type ConfigEnv,
|
||||||
type UserConfig,
|
type UserConfig,
|
||||||
@@ -33,8 +34,7 @@ export default ({
|
|||||||
// In future we might be able to do what is needed via code splitting at
|
// In future we might be able to do what is needed via code splitting at
|
||||||
// build time.
|
// build time.
|
||||||
process.env.VITE_PACKAGE = packageType ?? "full";
|
process.env.VITE_PACKAGE = packageType ?? "full";
|
||||||
|
const plugins: PluginOption[] = [
|
||||||
const plugins = [
|
|
||||||
react(),
|
react(),
|
||||||
svgrPlugin({
|
svgrPlugin({
|
||||||
svgrOptions: {
|
svgrOptions: {
|
||||||
|
|||||||
Reference in New Issue
Block a user