Compare commits
20 Commits
feat/ten-v
...
ten-vad-lo
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ea44b04bb3 | ||
|
|
c42274a511 | ||
|
|
9fc9655dbb | ||
|
|
e95e613c08 | ||
|
|
025735c490 | ||
|
|
dc1f30b84f | ||
|
|
dbd4eef899 | ||
|
|
325094b54d | ||
|
|
aff09d0e49 | ||
|
|
859db651e0 | ||
|
|
1ffee2d25e | ||
|
|
4a58277090 | ||
|
|
f2988cd689 | ||
|
|
b25cec3aa0 | ||
|
|
edd1e1d34e | ||
|
|
9f5b639190 | ||
|
|
428b76db25 | ||
|
|
0788e56c51 | ||
|
|
411e18c48a | ||
|
|
68d8bb1f92 |
@@ -31,6 +31,7 @@ interface Props {
|
||||
max: number;
|
||||
step: number;
|
||||
disabled?: boolean;
|
||||
tooltip?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -46,6 +47,7 @@ export const Slider: FC<Props> = ({
|
||||
max,
|
||||
step,
|
||||
disabled,
|
||||
tooltip = true,
|
||||
}) => {
|
||||
const onValueChange = useCallback(
|
||||
([v]: number[]) => onValueChangeProp(v),
|
||||
@@ -71,9 +73,13 @@ export const Slider: FC<Props> = ({
|
||||
<Range className={styles.highlight} />
|
||||
</Track>
|
||||
{/* Note: This is expected not to be visible on mobile.*/}
|
||||
<Tooltip placement="top" label={Math.round(value * 100).toString() + "%"}>
|
||||
{tooltip ? (
|
||||
<Tooltip placement="top" label={Math.round(value * 100).toString() + "%"}>
|
||||
<Thumb className={styles.handle} aria-label={label} />
|
||||
</Tooltip>
|
||||
) : (
|
||||
<Thumb className={styles.handle} aria-label={label} />
|
||||
</Tooltip>
|
||||
)}
|
||||
</Root>
|
||||
);
|
||||
};
|
||||
|
||||
@@ -74,6 +74,8 @@ export function LivekitRoomAudioRenderer({
|
||||
)
|
||||
// Only keep audio tracks
|
||||
.filter((ref) => ref.publication.kind === Track.Kind.Audio)
|
||||
// Never render local participant's own audio back to themselves
|
||||
.filter((ref) => !ref.participant.isLocal)
|
||||
// Only keep tracks from participants that are in the validIdentities list
|
||||
.filter((ref) => {
|
||||
const isValid = validIdentities.includes(ref.participant.identity);
|
||||
|
||||
@@ -23,13 +23,20 @@ declare function registerProcessor(
|
||||
processorCtor: new (...args: any[]) => AudioWorkletProcessor,
|
||||
): void;
|
||||
|
||||
interface TenVadParams {
|
||||
interface NoiseGateParams {
|
||||
noiseGateActive: boolean;
|
||||
threshold: number; // dBFS — gate opens above this, closes below it
|
||||
attackMs: number;
|
||||
holdMs: number;
|
||||
releaseMs: number;
|
||||
transientEnabled: boolean;
|
||||
transientThresholdDb: number; // dB above background RMS that triggers suppression
|
||||
transientReleaseMs: number; // how quickly suppression fades after transient ends
|
||||
// TEN-VAD params
|
||||
vadEnabled: boolean;
|
||||
vadPositiveThreshold: number; // open gate when prob >= this (0–1)
|
||||
vadNegativeThreshold: number; // close gate when prob < this (0–1)
|
||||
vadMode: "standard" | "aggressive" | "loose";
|
||||
holdMs: number; // hold time before closing gate (ms); 0 = no hold
|
||||
vadAggressive: boolean; // true: 1 ms open / 5 ms close; false: 5 ms / 20 ms
|
||||
}
|
||||
|
||||
interface VADGateMessage {
|
||||
@@ -37,6 +44,10 @@ interface VADGateMessage {
|
||||
open: boolean;
|
||||
}
|
||||
|
||||
function dbToLinear(db: number): number {
|
||||
return Math.pow(10, db / 20);
|
||||
}
|
||||
|
||||
/**
|
||||
* Thin synchronous wrapper around the TEN-VAD Emscripten WASM module.
|
||||
* Instantiated synchronously in the AudioWorklet constructor from a
|
||||
@@ -160,16 +171,42 @@ class TenVADRuntime {
|
||||
}
|
||||
|
||||
/**
|
||||
* AudioWorkletProcessor implementing an in-worklet TEN-VAD gate running
|
||||
* per-sample.
|
||||
* AudioWorkletProcessor implementing a noise gate, an optional transient
|
||||
* suppressor, and an optional in-worklet TEN-VAD gate — all running
|
||||
* per-sample in a single pass.
|
||||
*
|
||||
* Noise gate: opens when instantaneous peak exceeds threshold, closes below.
|
||||
* Attack, hold, and release times smooth the attenuation envelope.
|
||||
*
|
||||
* Transient suppressor: tracks a slow-moving RMS background level. When the
|
||||
* instantaneous peak exceeds the background by more than transientThresholdDb,
|
||||
* gain is instantly cut to 0 and releases over transientReleaseMs.
|
||||
*
|
||||
* TEN-VAD gate: accumulates audio with 3:1 decimation (48 kHz → 16 kHz),
|
||||
* runs the TEN-VAD model synchronously every 256 samples (16 ms), and
|
||||
* runs the TEN-VAD model synchronously every 160 samples (10 ms), and
|
||||
* controls vadGateOpen with hysteresis. No IPC round-trip required.
|
||||
* Asymmetric ramp: 5 ms open (minimise speech onset masking), 20 ms close
|
||||
* (de-click on silence).
|
||||
*/
|
||||
class TenVadProcessor extends AudioWorkletProcessor {
|
||||
class NoiseGateProcessor extends AudioWorkletProcessor {
|
||||
// Noise gate state
|
||||
private noiseGateActive = true;
|
||||
private threshold = dbToLinear(-60);
|
||||
private attackRate = 1.0 / (0.025 * sampleRate);
|
||||
private releaseRate = 1.0 / (0.15 * sampleRate);
|
||||
private holdTime = 0.2;
|
||||
private isOpen = false;
|
||||
private gateAttenuation = 0;
|
||||
private heldTime = 0;
|
||||
|
||||
// Transient suppressor state
|
||||
private transientEnabled = false;
|
||||
private transientRatio = dbToLinear(15);
|
||||
private transientReleaseRate = 1.0 / (0.08 * sampleRate);
|
||||
private transientAttenuation = 1.0;
|
||||
private slowRms = 0;
|
||||
private rmsCoeff = Math.exp(-1.0 / (0.2 * sampleRate));
|
||||
|
||||
// VAD gate state
|
||||
private vadGateOpen = true; // starts open; TEN-VAD closes it on first silent frame
|
||||
private vadAttenuation = 1.0;
|
||||
@@ -181,9 +218,7 @@ class TenVadProcessor extends AudioWorkletProcessor {
|
||||
private vadEnabled = false;
|
||||
private vadPositiveThreshold = 0.5;
|
||||
private vadNegativeThreshold = 0.3;
|
||||
private holdMs = 0;
|
||||
private vadHoldHops = 0; // hold expressed in VAD hops
|
||||
private vadHoldCounter = 0; // hops of continuous sub-threshold signal while gate is open
|
||||
private vadAggressive = false;
|
||||
private tenVadRuntime: TenVADRuntime | null = null;
|
||||
private tenVadModule: WebAssembly.Module | undefined = undefined;
|
||||
// 3:1 decimation from AudioContext sample rate to 16 kHz
|
||||
@@ -212,86 +247,89 @@ class TenVadProcessor extends AudioWorkletProcessor {
|
||||
this.tenVadRuntime = new TenVADRuntime(this.tenVadModule, 256, 0.5);
|
||||
this.port.postMessage({
|
||||
type: "log",
|
||||
msg: "[TenVad worklet] TEN-VAD runtime initialized, decRatio=" + this.decRatio,
|
||||
msg: "[NoiseGate worklet] TEN-VAD runtime initialized, decRatio=" + this.decRatio,
|
||||
});
|
||||
} catch (e) {
|
||||
this.port.postMessage({
|
||||
type: "log",
|
||||
msg: "[TenVad worklet] TEN-VAD init failed: " + String(e),
|
||||
msg: "[NoiseGate worklet] TEN-VAD init failed: " + String(e),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
this.port.onmessage = (
|
||||
e: MessageEvent<TenVadParams | VADGateMessage>,
|
||||
e: MessageEvent<NoiseGateParams | VADGateMessage>,
|
||||
): void => {
|
||||
if ((e.data as VADGateMessage).type === "vad-gate") {
|
||||
this.vadGateOpen = (e.data as VADGateMessage).open;
|
||||
} else {
|
||||
this.updateParams(e.data as TenVadParams);
|
||||
this.updateParams(e.data as NoiseGateParams);
|
||||
}
|
||||
};
|
||||
|
||||
this.updateParams({
|
||||
noiseGateActive: true,
|
||||
threshold: -60,
|
||||
attackMs: 25,
|
||||
holdMs: 200,
|
||||
releaseMs: 150,
|
||||
transientEnabled: false,
|
||||
transientThresholdDb: 15,
|
||||
transientReleaseMs: 80,
|
||||
vadEnabled: false,
|
||||
vadPositiveThreshold: 0.5,
|
||||
vadNegativeThreshold: 0.3,
|
||||
vadMode: "standard",
|
||||
holdMs: 0,
|
||||
vadAggressive: false,
|
||||
});
|
||||
|
||||
this.port.postMessage({
|
||||
type: "log",
|
||||
msg: "[TenVad worklet] constructor called, sampleRate=" + sampleRate,
|
||||
msg: "[NoiseGate worklet] constructor called, sampleRate=" + sampleRate,
|
||||
});
|
||||
}
|
||||
|
||||
private updateParams(p: TenVadParams): void {
|
||||
private updateParams(p: NoiseGateParams): void {
|
||||
this.noiseGateActive = p.noiseGateActive ?? true;
|
||||
this.threshold = dbToLinear(p.threshold);
|
||||
this.attackRate = 1.0 / ((p.attackMs / 1000) * sampleRate);
|
||||
this.releaseRate = 1.0 / ((p.releaseMs / 1000) * sampleRate);
|
||||
this.holdTime = p.holdMs / 1000;
|
||||
this.transientEnabled = p.transientEnabled;
|
||||
this.transientRatio = dbToLinear(p.transientThresholdDb);
|
||||
this.transientReleaseRate = 1.0 / ((p.transientReleaseMs / 1000) * sampleRate);
|
||||
this.vadEnabled = p.vadEnabled ?? false;
|
||||
this.vadPositiveThreshold = p.vadPositiveThreshold ?? 0.5;
|
||||
this.vadNegativeThreshold = p.vadNegativeThreshold ?? 0.3;
|
||||
this.holdMs = p.holdMs ?? 0;
|
||||
|
||||
const newMode = p.vadMode ?? "standard";
|
||||
if (newMode === "aggressive") {
|
||||
this.vadOpenRampRate = 1.0 / (0.001 * sampleRate); // 1 ms
|
||||
const newAggressive = p.vadAggressive ?? false;
|
||||
if (newAggressive) {
|
||||
this.vadOpenRampRate = 1.0 / (0.001 * sampleRate); // 1 ms — near-instant
|
||||
this.vadCloseRampRate = 1.0 / (0.005 * sampleRate); // 5 ms
|
||||
} else if (newMode === "loose") {
|
||||
this.vadOpenRampRate = 1.0 / (0.012 * sampleRate); // 12 ms
|
||||
this.vadCloseRampRate = 1.0 / (0.032 * sampleRate); // 32 ms
|
||||
} else {
|
||||
this.vadOpenRampRate = 1.0 / (0.005 * sampleRate); // 5 ms
|
||||
this.vadCloseRampRate = 1.0 / (0.02 * sampleRate); // 20 ms
|
||||
}
|
||||
|
||||
// Hop size: aggressive=160 (10 ms @ 16 kHz), others=256 (16 ms)
|
||||
const newHopSize = newMode === "aggressive" ? 160 : 256;
|
||||
if (newHopSize !== this.vadHopSize && this.tenVadModule) {
|
||||
// Recreate runtime if mode changed (hop size differs between standard/aggressive)
|
||||
const newHopSize = newAggressive ? 160 : 256;
|
||||
if (newAggressive !== this.vadAggressive && this.tenVadModule) {
|
||||
this.tenVadRuntime?.destroy();
|
||||
this.tenVadRuntime = null;
|
||||
this.vadHopCount = 0;
|
||||
try {
|
||||
this.tenVadRuntime = new TenVADRuntime(this.tenVadModule, newHopSize, 0.5);
|
||||
} catch (e) {
|
||||
this.port.postMessage({ type: "log", msg: "[TenVad worklet] TEN-VAD recreate failed: " + String(e) });
|
||||
this.port.postMessage({ type: "log", msg: "[NoiseGate worklet] TEN-VAD recreate failed: " + String(e) });
|
||||
}
|
||||
}
|
||||
this.vadAggressive = newAggressive;
|
||||
this.vadHopSize = newHopSize;
|
||||
|
||||
// Recompute hold in hops: ceil((holdMs / 1000) * 16000 / vadHopSize)
|
||||
this.vadHoldHops = this.holdMs > 0
|
||||
? Math.ceil((this.holdMs / 1000) * 16000 / this.vadHopSize)
|
||||
: 0;
|
||||
this.vadHoldCounter = 0;
|
||||
|
||||
// When VAD is disabled, open the gate immediately
|
||||
if (!this.vadEnabled) this.vadGateOpen = true;
|
||||
this.port.postMessage({
|
||||
type: "log",
|
||||
msg: "[TenVad worklet] params updated: vadEnabled=" + p.vadEnabled
|
||||
msg: "[NoiseGate worklet] params updated: threshold=" + p.threshold
|
||||
+ " vadEnabled=" + p.vadEnabled
|
||||
+ " vadPos=" + p.vadPositiveThreshold
|
||||
+ " vadNeg=" + p.vadNegativeThreshold
|
||||
+ " vadMode=" + newMode
|
||||
+ " holdMs=" + this.holdMs,
|
||||
+ " vadNeg=" + p.vadNegativeThreshold,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -300,9 +338,63 @@ class TenVadProcessor extends AudioWorkletProcessor {
|
||||
const output = outputs[0];
|
||||
if (!input || input.length === 0) return true;
|
||||
|
||||
const channels = input.length;
|
||||
const blockSize = input[0]?.length ?? 128;
|
||||
const samplePeriod = 1.0 / sampleRate;
|
||||
|
||||
for (let i = 0; i < blockSize; i++) {
|
||||
// Peak detection across all channels
|
||||
let curLevel = Math.abs(input[0]?.[i] ?? 0);
|
||||
for (let j = 1; j < channels; j++) {
|
||||
curLevel = Math.max(curLevel, Math.abs(input[j]?.[i] ?? 0));
|
||||
}
|
||||
|
||||
// --- Transient suppressor ---
|
||||
let transientGain = 1.0;
|
||||
if (this.transientEnabled) {
|
||||
this.slowRms = Math.sqrt(
|
||||
this.rmsCoeff * this.slowRms * this.slowRms +
|
||||
(1.0 - this.rmsCoeff) * curLevel * curLevel,
|
||||
);
|
||||
const background = Math.max(this.slowRms, 1e-6);
|
||||
if (curLevel > background * this.transientRatio) {
|
||||
this.transientAttenuation = 0.0;
|
||||
} else {
|
||||
this.transientAttenuation = Math.min(
|
||||
1.0,
|
||||
this.transientAttenuation + this.transientReleaseRate,
|
||||
);
|
||||
}
|
||||
transientGain = this.transientAttenuation;
|
||||
}
|
||||
|
||||
// --- Noise gate ---
|
||||
if (this.noiseGateActive) {
|
||||
if (curLevel > this.threshold && !this.isOpen) {
|
||||
this.isOpen = true;
|
||||
}
|
||||
if (curLevel <= this.threshold && this.isOpen) {
|
||||
this.heldTime = 0;
|
||||
this.isOpen = false;
|
||||
}
|
||||
if (this.isOpen) {
|
||||
this.gateAttenuation = Math.min(
|
||||
1.0,
|
||||
this.gateAttenuation + this.attackRate,
|
||||
);
|
||||
} else {
|
||||
this.heldTime += samplePeriod;
|
||||
if (this.heldTime > this.holdTime) {
|
||||
this.gateAttenuation = Math.max(
|
||||
0.0,
|
||||
this.gateAttenuation - this.releaseRate,
|
||||
);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
this.gateAttenuation = 1.0;
|
||||
}
|
||||
|
||||
// --- TEN-VAD in-worklet processing ---
|
||||
// Accumulate raw mono samples with decRatio:1 decimation (48 kHz → 16 kHz).
|
||||
// Every 256 output samples (16 ms) run the WASM VAD and update vadGateOpen.
|
||||
@@ -325,25 +417,10 @@ class TenVadProcessor extends AudioWorkletProcessor {
|
||||
if (this.vadHopCount >= this.vadHopSize) {
|
||||
this.vadHopCount = 0;
|
||||
const prob = this.tenVadRuntime.process(this.vadHopBuf);
|
||||
if (prob >= this.vadPositiveThreshold) {
|
||||
// Speech detected — open gate, reset hold counter
|
||||
if (!this.vadGateOpen && prob >= this.vadPositiveThreshold) {
|
||||
this.vadGateOpen = true;
|
||||
this.vadHoldCounter = 0;
|
||||
} else if (prob < this.vadNegativeThreshold) {
|
||||
if (this.vadGateOpen) {
|
||||
if (this.vadHoldHops === 0) {
|
||||
this.vadGateOpen = false;
|
||||
} else {
|
||||
this.vadHoldCounter++;
|
||||
if (this.vadHoldCounter >= this.vadHoldHops) {
|
||||
this.vadGateOpen = false;
|
||||
this.vadHoldCounter = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Ambiguous zone — reset hold counter so hold only fires on sustained silence
|
||||
this.vadHoldCounter = 0;
|
||||
} else if (this.vadGateOpen && prob < this.vadNegativeThreshold) {
|
||||
this.vadGateOpen = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -364,7 +441,7 @@ class TenVadProcessor extends AudioWorkletProcessor {
|
||||
);
|
||||
}
|
||||
|
||||
const gain = this.vadAttenuation;
|
||||
const gain = this.gateAttenuation * transientGain * this.vadAttenuation;
|
||||
|
||||
for (let c = 0; c < output.length; c++) {
|
||||
const inCh = input[c] ?? input[0];
|
||||
@@ -379,7 +456,10 @@ class TenVadProcessor extends AudioWorkletProcessor {
|
||||
if (this.logCounter % 375 === 0) {
|
||||
this.port.postMessage({
|
||||
type: "log",
|
||||
msg: "[TenVad worklet] vadOpen=" + this.vadGateOpen
|
||||
msg: "[NoiseGate worklet] gateOpen=" + this.isOpen
|
||||
+ " gateAtten=" + this.gateAttenuation.toFixed(3)
|
||||
+ " transientAtten=" + this.transientAttenuation.toFixed(3)
|
||||
+ " vadOpen=" + this.vadGateOpen
|
||||
+ " vadAtten=" + this.vadAttenuation.toFixed(3),
|
||||
});
|
||||
}
|
||||
@@ -388,4 +468,4 @@ class TenVadProcessor extends AudioWorkletProcessor {
|
||||
}
|
||||
}
|
||||
|
||||
registerProcessor("ten-vad-processor", TenVadProcessor);
|
||||
registerProcessor("noise-gate-processor", NoiseGateProcessor);
|
||||
@@ -9,17 +9,24 @@ import { type Track } from "livekit-client";
|
||||
import { logger } from "matrix-js-sdk/lib/logger";
|
||||
// ?worker&url tells Vite to compile the TypeScript worklet and return its URL.
|
||||
// Without this, Vite copies the .ts file verbatim and the browser rejects it.
|
||||
import compiledWorkletUrl from "./TenVadProcessor.worklet.ts?worker&url";
|
||||
import compiledWorkletUrl from "./NoiseGateProcessor.worklet.ts?worker&url";
|
||||
|
||||
const log = logger.getChild("[TenVadTransformer]");
|
||||
const log = logger.getChild("[NoiseGateTransformer]");
|
||||
|
||||
export interface TenVadParams {
|
||||
export interface NoiseGateParams {
|
||||
noiseGateActive: boolean;
|
||||
threshold: number; // dBFS — gate opens above this, closes below it
|
||||
attackMs: number;
|
||||
holdMs: number;
|
||||
releaseMs: number;
|
||||
transientEnabled: boolean;
|
||||
transientThresholdDb: number; // dB above background RMS that triggers suppression
|
||||
transientReleaseMs: number; // ms for suppression to fade after transient ends
|
||||
// TEN-VAD params — processed entirely inside the AudioWorklet
|
||||
vadEnabled: boolean;
|
||||
vadPositiveThreshold: number; // open gate when prob >= this (0–1)
|
||||
vadNegativeThreshold: number; // close gate when prob < this (0–1); computed by Publisher
|
||||
vadMode: "standard" | "aggressive" | "loose";
|
||||
holdMs: number; // hold time before closing gate (ms); 0 = no hold
|
||||
vadPositiveThreshold: number; // open gate when isSpeech prob >= this (0–1)
|
||||
vadNegativeThreshold: number; // close gate when isSpeech prob < this (0–1)
|
||||
vadAggressive: boolean; // true: 1 ms open / 5 ms close ramp; false: 5 ms / 20 ms
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -65,7 +72,8 @@ function getTenVADModule(): Promise<WebAssembly.Module> {
|
||||
}
|
||||
|
||||
/**
|
||||
* LiveKit audio track processor that applies TEN-VAD via AudioWorklet.
|
||||
* LiveKit audio track processor that applies a noise gate, optional transient
|
||||
* suppressor, and optional TEN-VAD gate via AudioWorklet.
|
||||
*
|
||||
* The TEN-VAD WASM module is fetched once, compiled, and passed to the worklet
|
||||
* via processorOptions so it runs synchronously inside the audio thread —
|
||||
@@ -74,16 +82,16 @@ function getTenVADModule(): Promise<WebAssembly.Module> {
|
||||
* Audio graph: sourceNode → workletNode → destinationNode
|
||||
* processedTrack is destinationNode.stream.getAudioTracks()[0]
|
||||
*/
|
||||
export class TenVadTransformer implements AudioTrackProcessor {
|
||||
public readonly name = "ten-vad";
|
||||
export class NoiseGateTransformer implements AudioTrackProcessor {
|
||||
public readonly name = "noise-gate";
|
||||
public processedTrack?: MediaStreamTrack;
|
||||
|
||||
private workletNode?: AudioWorkletNode;
|
||||
private sourceNode?: MediaStreamAudioSourceNode;
|
||||
private destinationNode?: MediaStreamAudioDestinationNode;
|
||||
private params: TenVadParams;
|
||||
private params: NoiseGateParams;
|
||||
|
||||
public constructor(params: TenVadParams) {
|
||||
public constructor(params: NoiseGateParams) {
|
||||
this.params = { ...params };
|
||||
}
|
||||
|
||||
@@ -107,7 +115,7 @@ export class TenVadTransformer implements AudioTrackProcessor {
|
||||
|
||||
this.workletNode = new AudioWorkletNode(
|
||||
audioContext,
|
||||
"ten-vad-processor",
|
||||
"noise-gate-processor",
|
||||
{
|
||||
processorOptions: {
|
||||
tenVadModule,
|
||||
@@ -150,7 +158,7 @@ export class TenVadTransformer implements AudioTrackProcessor {
|
||||
}
|
||||
|
||||
/** Push updated gate/VAD parameters to the running worklet. */
|
||||
public updateParams(params: TenVadParams): void {
|
||||
public updateParams(params: NoiseGateParams): void {
|
||||
this.params = { ...params };
|
||||
this.sendParams();
|
||||
}
|
||||
87
src/settings/NoiseLevelSlider.module.css
Normal file
87
src/settings/NoiseLevelSlider.module.css
Normal file
@@ -0,0 +1,87 @@
|
||||
/*
|
||||
Copyright 2026 New Vector Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
.wrapper {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
position: relative;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.slider {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
position: relative;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.track {
|
||||
flex-grow: 1;
|
||||
border-radius: var(--cpd-radius-pill-effect);
|
||||
height: var(--cpd-space-4x);
|
||||
outline: var(--cpd-border-width-1) solid var(--cpd-color-border-interactive-primary);
|
||||
outline-offset: calc(-1 * var(--cpd-border-width-1));
|
||||
cursor: pointer;
|
||||
position: relative;
|
||||
overflow: hidden;
|
||||
/* Base background */
|
||||
background: var(--cpd-color-bg-subtle-primary);
|
||||
}
|
||||
|
||||
/* Live mic level fill — driven by --mic-level CSS variable */
|
||||
.track::before {
|
||||
content: "";
|
||||
position: absolute;
|
||||
inset: 0;
|
||||
width: var(--mic-level, 0%);
|
||||
background: var(--cpd-color-gray-600, #808080);
|
||||
opacity: 0.55;
|
||||
border-radius: var(--cpd-radius-pill-effect);
|
||||
transition: width 40ms linear;
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
/* Green when mic level is at or above the threshold */
|
||||
.track[data-active="true"]::before {
|
||||
background: var(--cpd-color-green-900, #1a7f4b);
|
||||
}
|
||||
|
||||
/* Threshold marker — driven by --threshold-pct CSS variable */
|
||||
.track::after {
|
||||
content: "";
|
||||
position: absolute;
|
||||
top: 0;
|
||||
bottom: 0;
|
||||
left: var(--threshold-pct, 50%);
|
||||
width: 2px;
|
||||
background: var(--cpd-color-text-primary);
|
||||
opacity: 0.6;
|
||||
pointer-events: none;
|
||||
transform: translateX(-50%);
|
||||
}
|
||||
|
||||
/* Hide the Radix Range highlight — we use our own visuals */
|
||||
.range {
|
||||
display: none;
|
||||
}
|
||||
|
||||
.handle {
|
||||
display: block;
|
||||
block-size: var(--cpd-space-5x);
|
||||
inline-size: var(--cpd-space-5x);
|
||||
border-radius: var(--cpd-radius-pill-effect);
|
||||
background: var(--cpd-color-bg-action-primary-rest);
|
||||
box-shadow: 0 0 0 2px var(--cpd-color-bg-canvas-default);
|
||||
cursor: pointer;
|
||||
transition: background-color ease 0.15s;
|
||||
z-index: 1;
|
||||
}
|
||||
|
||||
.handle:focus-visible {
|
||||
outline: 2px solid var(--cpd-color-border-focused);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
139
src/settings/NoiseLevelSlider.tsx
Normal file
139
src/settings/NoiseLevelSlider.tsx
Normal file
@@ -0,0 +1,139 @@
|
||||
/*
|
||||
Copyright 2026 New Vector Ltd.
|
||||
|
||||
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { Root, Track, Range, Thumb } from "@radix-ui/react-slider";
|
||||
import { type FC, useCallback, useEffect, useRef } from "react";
|
||||
|
||||
import styles from "./NoiseLevelSlider.module.css";
|
||||
|
||||
interface Props {
|
||||
label: string;
|
||||
value: number;
|
||||
onValueChange: (value: number) => void;
|
||||
onValueCommit?: (value: number) => void;
|
||||
min: number;
|
||||
max: number;
|
||||
step: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Threshold slider that shows live microphone input level as a background fill,
|
||||
* similar to Discord's input sensitivity control.
|
||||
*
|
||||
* The green fill represents your current mic volume in real time.
|
||||
* Drag the handle to set the gate threshold — audio below it will be silenced.
|
||||
*/
|
||||
export const NoiseLevelSlider: FC<Props> = ({
|
||||
label,
|
||||
value,
|
||||
onValueChange: onValueChangeProp,
|
||||
onValueCommit: onValueCommitProp,
|
||||
min,
|
||||
max,
|
||||
step,
|
||||
}) => {
|
||||
const trackRef = useRef<HTMLSpanElement>(null);
|
||||
const animFrameRef = useRef<number>(0);
|
||||
const analyserRef = useRef<AnalyserNode | null>(null);
|
||||
const streamRef = useRef<MediaStream | null>(null);
|
||||
const dataRef = useRef<Float32Array<ArrayBuffer> | null>(null);
|
||||
const thresholdPctRef = useRef<number>(0);
|
||||
|
||||
// Start mic monitoring via AnalyserNode
|
||||
useEffect(() => {
|
||||
let ctx: AudioContext | null = null;
|
||||
|
||||
navigator.mediaDevices
|
||||
.getUserMedia({ audio: true, video: false })
|
||||
.then((stream) => {
|
||||
streamRef.current = stream;
|
||||
ctx = new AudioContext();
|
||||
const source = ctx.createMediaStreamSource(stream);
|
||||
const analyser = ctx.createAnalyser();
|
||||
analyser.fftSize = 1024;
|
||||
analyser.smoothingTimeConstant = 0.6;
|
||||
source.connect(analyser);
|
||||
analyserRef.current = analyser;
|
||||
dataRef.current = new Float32Array(analyser.fftSize) as Float32Array<ArrayBuffer>;
|
||||
})
|
||||
.catch(() => {
|
||||
// Mic not available — live level stays at 0
|
||||
});
|
||||
|
||||
return (): void => {
|
||||
cancelAnimationFrame(animFrameRef.current);
|
||||
streamRef.current?.getTracks().forEach((t) => t.stop());
|
||||
streamRef.current = null;
|
||||
analyserRef.current = null;
|
||||
void ctx?.close();
|
||||
};
|
||||
}, []);
|
||||
|
||||
// rAF loop — reads RMS level and updates CSS variable on the track element
|
||||
useEffect(() => {
|
||||
const tick = (): void => {
|
||||
animFrameRef.current = requestAnimationFrame(tick);
|
||||
|
||||
const analyser = analyserRef.current;
|
||||
const data = dataRef.current;
|
||||
const track = trackRef.current;
|
||||
if (!analyser || !data || !track) return;
|
||||
|
||||
analyser.getFloatTimeDomainData(data);
|
||||
|
||||
// Peak detection — matches the gate's own measurement method
|
||||
let peak = 0;
|
||||
for (const s of data) peak = Math.max(peak, Math.abs(s));
|
||||
const dbfs = peak > 0 ? 20 * Math.log10(peak) : -Infinity;
|
||||
|
||||
// Map dBFS value to slider percentage (same scale as min/max)
|
||||
const clampedDb = Math.max(min, Math.min(max, dbfs));
|
||||
const levelPct = ((clampedDb - min) / (max - min)) * 100;
|
||||
|
||||
track.style.setProperty("--mic-level", `${levelPct.toFixed(1)}%`);
|
||||
track.dataset.active = String(levelPct >= thresholdPctRef.current);
|
||||
};
|
||||
|
||||
animFrameRef.current = requestAnimationFrame(tick);
|
||||
return (): void => cancelAnimationFrame(animFrameRef.current);
|
||||
}, [min, max]);
|
||||
|
||||
// Keep threshold marker in sync with slider value
|
||||
useEffect(() => {
|
||||
const track = trackRef.current;
|
||||
if (!track) return;
|
||||
const thresholdPct = ((value - min) / (max - min)) * 100;
|
||||
thresholdPctRef.current = thresholdPct;
|
||||
track.style.setProperty("--threshold-pct", `${thresholdPct.toFixed(1)}%`);
|
||||
}, [value, min, max]);
|
||||
|
||||
const onValueChange = useCallback(
|
||||
([v]: number[]) => onValueChangeProp(v),
|
||||
[onValueChangeProp],
|
||||
);
|
||||
const onValueCommit = useCallback(
|
||||
([v]: number[]) => onValueCommitProp?.(v),
|
||||
[onValueCommitProp],
|
||||
);
|
||||
|
||||
return (
|
||||
<Root
|
||||
className={styles.slider}
|
||||
value={[value]}
|
||||
onValueChange={onValueChange}
|
||||
onValueCommit={onValueCommit}
|
||||
min={min}
|
||||
max={max}
|
||||
step={step}
|
||||
>
|
||||
<Track className={styles.track} ref={trackRef}>
|
||||
<Range className={styles.range} />
|
||||
</Track>
|
||||
<Thumb className={styles.handle} aria-label={label} />
|
||||
</Root>
|
||||
);
|
||||
};
|
||||
@@ -35,30 +35,37 @@ Please see LICENSE in the repository root for full details.
|
||||
color: var(--cpd-color-text-secondary);
|
||||
}
|
||||
|
||||
.vadSection {
|
||||
.noiseGateSection {
|
||||
margin-block-start: var(--cpd-space-6x);
|
||||
}
|
||||
|
||||
.vadHeading {
|
||||
.noiseGateHeading {
|
||||
color: var(--cpd-color-text-secondary);
|
||||
margin-block: var(--cpd-space-3x) 0;
|
||||
}
|
||||
|
||||
.vadSeparator {
|
||||
.thresholdSlider {
|
||||
margin-block-start: calc(-32px + var(--cpd-space-2x));
|
||||
}
|
||||
|
||||
.noiseGateSeparator {
|
||||
margin-block: 6px var(--cpd-space-4x);
|
||||
}
|
||||
|
||||
.vadRampLabel {
|
||||
display: block;
|
||||
margin-block: var(--cpd-space-6x) var(--cpd-space-1x);
|
||||
.advancedGate {
|
||||
margin-top: var(--cpd-space-3x);
|
||||
}
|
||||
|
||||
.vadRampForm {
|
||||
margin-top: 0;
|
||||
.advancedGateToggle {
|
||||
all: unset;
|
||||
cursor: pointer;
|
||||
font: var(--cpd-font-body-sm-semibold);
|
||||
color: var(--cpd-color-text-secondary);
|
||||
user-select: none;
|
||||
}
|
||||
|
||||
.vadSpacedSlider {
|
||||
margin-block-start: var(--cpd-space-6x);
|
||||
.advancedGateToggle:hover {
|
||||
color: var(--cpd-color-text-primary);
|
||||
}
|
||||
|
||||
.restoreDefaults {
|
||||
|
||||
@@ -5,7 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
|
||||
Please see LICENSE in the repository root for full details.
|
||||
*/
|
||||
|
||||
import { type FC, type ReactNode, useEffect, useId, useState } from "react";
|
||||
import { type ChangeEvent, type FC, type ReactNode, useEffect, useId, useState, useCallback } from "react";
|
||||
import { useTranslation } from "react-i18next";
|
||||
import { type MatrixClient } from "matrix-js-sdk";
|
||||
import {
|
||||
@@ -33,16 +33,22 @@ import {
|
||||
soundEffectVolume as soundEffectVolumeSetting,
|
||||
backgroundBlur as backgroundBlurSetting,
|
||||
developerMode,
|
||||
noiseGateEnabled as noiseGateEnabledSetting,
|
||||
noiseGateThreshold as noiseGateThresholdSetting,
|
||||
noiseGateAttack as noiseGateAttackSetting,
|
||||
noiseGateHold as noiseGateHoldSetting,
|
||||
noiseGateRelease as noiseGateReleaseSetting,
|
||||
transientSuppressorEnabled as transientSuppressorEnabledSetting,
|
||||
transientThreshold as transientThresholdSetting,
|
||||
transientRelease as transientReleaseSetting,
|
||||
vadEnabled as vadEnabledSetting,
|
||||
vadPositiveThreshold as vadPositiveThresholdSetting,
|
||||
vadNegativeThreshold as vadNegativeThresholdSetting,
|
||||
vadMode as vadModeSetting,
|
||||
vadAdvancedEnabled as vadAdvancedEnabledSetting,
|
||||
vadAdvancedOpenThreshold as vadAdvancedOpenThresholdSetting,
|
||||
vadAdvancedCloseThreshold as vadAdvancedCloseThresholdSetting,
|
||||
vadHoldTime as vadHoldTimeSetting,
|
||||
} from "./settings";
|
||||
import { PreferencesSettingsTab } from "./PreferencesSettingsTab";
|
||||
import { Slider } from "../Slider";
|
||||
import { NoiseLevelSlider } from "./NoiseLevelSlider";
|
||||
import { DeviceSelection } from "./DeviceSelection";
|
||||
import { useTrackProcessor } from "../livekit/TrackProcessorContext";
|
||||
import { DeveloperSettingsTab } from "./DeveloperSettingsTab";
|
||||
@@ -123,25 +129,50 @@ export const SettingsModal: FC<Props> = ({
|
||||
const [soundVolumeRaw, setSoundVolumeRaw] = useState(soundVolume);
|
||||
const [showDeveloperSettingsTab] = useSetting(developerMode);
|
||||
|
||||
// Noise gate settings
|
||||
const [noiseGateEnabled, setNoiseGateEnabled] = useSetting(noiseGateEnabledSetting);
|
||||
const [noiseGateThreshold, setNoiseGateThreshold] = useSetting(noiseGateThresholdSetting);
|
||||
const [noiseGateThresholdRaw, setNoiseGateThresholdRaw] = useState(noiseGateThreshold);
|
||||
const [noiseGateAttack, setNoiseGateAttack] = useSetting(noiseGateAttackSetting);
|
||||
const [noiseGateAttackRaw, setNoiseGateAttackRaw] = useState(noiseGateAttack);
|
||||
const [noiseGateHold, setNoiseGateHold] = useSetting(noiseGateHoldSetting);
|
||||
const [noiseGateHoldRaw, setNoiseGateHoldRaw] = useState(noiseGateHold);
|
||||
const [noiseGateRelease, setNoiseGateRelease] = useSetting(noiseGateReleaseSetting);
|
||||
const [noiseGateReleaseRaw, setNoiseGateReleaseRaw] = useState(noiseGateRelease);
|
||||
|
||||
const [showAdvancedGate, setShowAdvancedGate] = useState(false);
|
||||
|
||||
// Voice activity detection
|
||||
const vadStateGroup = useId();
|
||||
const vadModeRadioGroup = useId();
|
||||
const [vadActive, setVadActive] = useSetting(vadEnabledSetting);
|
||||
const [vadSensitivity, setVadSensitivity] = useSetting(vadPositiveThresholdSetting);
|
||||
const [vadSensitivityRaw, setVadSensitivityRaw] = useState(vadSensitivity);
|
||||
const [vadAdvanced, setVadAdvanced] = useSetting(vadAdvancedEnabledSetting);
|
||||
const vadState = !vadActive ? "disabled" : vadAdvanced ? "advanced" : "simple";
|
||||
const setVadState = (s: "disabled" | "simple" | "advanced"): void => {
|
||||
setVadActive(s !== "disabled");
|
||||
setVadAdvanced(s === "advanced");
|
||||
};
|
||||
const [vadModeValue, setVadModeValue] = useSetting(vadModeSetting);
|
||||
const [vadAdvOpen, setVadAdvOpen] = useSetting(vadAdvancedOpenThresholdSetting);
|
||||
const [vadAdvOpenRaw, setVadAdvOpenRaw] = useState(vadAdvOpen);
|
||||
const [vadAdvClose, setVadAdvClose] = useSetting(vadAdvancedCloseThresholdSetting);
|
||||
const [vadAdvCloseRaw, setVadAdvCloseRaw] = useState(vadAdvClose);
|
||||
const [vadHold, setVadHold] = useSetting(vadHoldTimeSetting);
|
||||
const [vadHoldRaw, setVadHoldRaw] = useState(vadHold);
|
||||
const [vadPositiveThreshold, setVadPositiveThreshold] = useSetting(vadPositiveThresholdSetting);
|
||||
const [vadPositiveThresholdRaw, setVadPositiveThresholdRaw] = useState(vadPositiveThreshold);
|
||||
const [vadNegativeThreshold, setVadNegativeThreshold] = useSetting(vadNegativeThresholdSetting);
|
||||
const [vadNegativeThresholdRaw, setVadNegativeThresholdRaw] = useState(vadNegativeThreshold);
|
||||
|
||||
// Transient suppressor settings
|
||||
const [transientEnabled, setTransientEnabled] = useSetting(transientSuppressorEnabledSetting);
|
||||
const [transientThreshold, setTransientThreshold] = useSetting(transientThresholdSetting);
|
||||
const [transientThresholdRaw, setTransientThresholdRaw] = useState(transientThreshold);
|
||||
const [transientRelease, setTransientRelease] = useSetting(transientReleaseSetting);
|
||||
const [transientReleaseRaw, setTransientReleaseRaw] = useState(transientRelease);
|
||||
|
||||
const resetTransientDefaults = useCallback((): void => {
|
||||
const t = transientThresholdSetting.defaultValue;
|
||||
const r = transientReleaseSetting.defaultValue;
|
||||
setTransientThreshold(t); setTransientThresholdRaw(t);
|
||||
setTransientRelease(r); setTransientReleaseRaw(r);
|
||||
}, [setTransientThreshold, setTransientRelease]);
|
||||
|
||||
const resetGateDefaults = useCallback((): void => {
|
||||
const a = noiseGateAttackSetting.defaultValue;
|
||||
const h = noiseGateHoldSetting.defaultValue;
|
||||
const r = noiseGateReleaseSetting.defaultValue;
|
||||
setNoiseGateAttack(a); setNoiseGateAttackRaw(a);
|
||||
setNoiseGateHold(h); setNoiseGateHoldRaw(h);
|
||||
setNoiseGateRelease(r); setNoiseGateReleaseRaw(r);
|
||||
}, [setNoiseGateAttack, setNoiseGateHold, setNoiseGateRelease]);
|
||||
|
||||
const { available: isRageshakeAvailable } = useSubmitRageshake();
|
||||
|
||||
@@ -201,185 +232,262 @@ export const SettingsModal: FC<Props> = ({
|
||||
/>
|
||||
</div>
|
||||
</Form>
|
||||
<div className={styles.vadSection}>
|
||||
<div className={styles.noiseGateSection}>
|
||||
<Heading
|
||||
type="body"
|
||||
weight="semibold"
|
||||
size="sm"
|
||||
as="h4"
|
||||
className={styles.vadHeading}
|
||||
className={styles.noiseGateHeading}
|
||||
>
|
||||
Voice Activity Detection
|
||||
Noise Gate
|
||||
</Heading>
|
||||
<Separator className={styles.vadSeparator} />
|
||||
<Form>
|
||||
<InlineField
|
||||
name={vadStateGroup}
|
||||
control={
|
||||
<RadioControl
|
||||
checked={vadState === "disabled"}
|
||||
value="disabled"
|
||||
onChange={(): void => setVadState("disabled")}
|
||||
/>
|
||||
}
|
||||
>
|
||||
<Label>Disabled</Label>
|
||||
</InlineField>
|
||||
<InlineField
|
||||
name={vadStateGroup}
|
||||
control={
|
||||
<RadioControl
|
||||
checked={vadState === "simple"}
|
||||
value="simple"
|
||||
onChange={(): void => setVadState("simple")}
|
||||
/>
|
||||
}
|
||||
>
|
||||
<Label>Simple</Label>
|
||||
</InlineField>
|
||||
<InlineField
|
||||
name={vadStateGroup}
|
||||
control={
|
||||
<RadioControl
|
||||
checked={vadState === "advanced"}
|
||||
value="advanced"
|
||||
onChange={(): void => setVadState("advanced")}
|
||||
/>
|
||||
}
|
||||
>
|
||||
<Label>Advanced</Label>
|
||||
</InlineField>
|
||||
</Form>
|
||||
{vadState !== "disabled" && (
|
||||
<>
|
||||
{vadState === "simple" && (
|
||||
<div className={styles.volumeSlider}>
|
||||
<span className={styles.sliderLabel}>
|
||||
Sensitivity: {Math.round(vadSensitivityRaw * 100)}%
|
||||
</span>
|
||||
<p>Higher values require more confident speech detection before opening.</p>
|
||||
<Slider
|
||||
label="VAD sensitivity"
|
||||
value={vadSensitivityRaw}
|
||||
onValueChange={setVadSensitivityRaw}
|
||||
onValueCommit={setVadSensitivity}
|
||||
min={0.1}
|
||||
max={1.0}
|
||||
step={0.05}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
{vadState === "advanced" && (
|
||||
<Separator className={styles.noiseGateSeparator} />
|
||||
<FieldRow>
|
||||
<InputField
|
||||
id="noiseGateEnabled"
|
||||
type="checkbox"
|
||||
label="Enable noise gate"
|
||||
description="Suppress audio below a configurable threshold."
|
||||
checked={noiseGateEnabled}
|
||||
onChange={(e: ChangeEvent<HTMLInputElement>): void =>
|
||||
setNoiseGateEnabled(e.target.checked)
|
||||
}
|
||||
/>
|
||||
</FieldRow>
|
||||
{noiseGateEnabled && (
|
||||
<>
|
||||
<div className={`${styles.volumeSlider} ${styles.thresholdSlider}`}>
|
||||
<span className={styles.sliderLabel}>Threshold</span>
|
||||
<p>Gate opens above this level, closes below it.</p>
|
||||
<NoiseLevelSlider
|
||||
label="Noise gate threshold"
|
||||
value={noiseGateThresholdRaw}
|
||||
onValueChange={setNoiseGateThresholdRaw}
|
||||
onValueCommit={setNoiseGateThreshold}
|
||||
min={-100}
|
||||
max={0}
|
||||
step={1}
|
||||
/>
|
||||
</div>
|
||||
<div className={styles.advancedGate}>
|
||||
<button
|
||||
className={styles.advancedGateToggle}
|
||||
onClick={(): void => setShowAdvancedGate((v) => !v)}
|
||||
>
|
||||
{showAdvancedGate ? "▾" : "▸"} Advanced settings
|
||||
</button>
|
||||
{showAdvancedGate && (
|
||||
<>
|
||||
<span className={styles.vadRampLabel}>Ramp profiles</span>
|
||||
<Form className={styles.vadRampForm}>
|
||||
<InlineField
|
||||
name={vadModeRadioGroup}
|
||||
control={
|
||||
<RadioControl
|
||||
checked={vadModeValue === "loose"}
|
||||
value="loose"
|
||||
onChange={(): void => setVadModeValue("loose")}
|
||||
/>
|
||||
}
|
||||
>
|
||||
<Label>Loose</Label>
|
||||
<HelpMessage>256 samples / 16 ms — 12 ms open / 32 ms close ramp.</HelpMessage>
|
||||
</InlineField>
|
||||
<InlineField
|
||||
name={vadModeRadioGroup}
|
||||
control={
|
||||
<RadioControl
|
||||
checked={vadModeValue === "standard"}
|
||||
value="standard"
|
||||
onChange={(): void => setVadModeValue("standard")}
|
||||
/>
|
||||
}
|
||||
>
|
||||
<Label>Standard</Label>
|
||||
<HelpMessage>256 samples / 16 ms — 5 ms open / 20 ms close ramp.</HelpMessage>
|
||||
</InlineField>
|
||||
<InlineField
|
||||
name={vadModeRadioGroup}
|
||||
control={
|
||||
<RadioControl
|
||||
checked={vadModeValue === "aggressive"}
|
||||
value="aggressive"
|
||||
onChange={(): void => setVadModeValue("aggressive")}
|
||||
/>
|
||||
}
|
||||
>
|
||||
<Label>Aggressive</Label>
|
||||
<HelpMessage>160 samples / 10 ms — 1 ms open / 5 ms close ramp.</HelpMessage>
|
||||
</InlineField>
|
||||
</Form>
|
||||
<div className={`${styles.volumeSlider} ${styles.vadSpacedSlider}`}>
|
||||
<span className={styles.sliderLabel}>
|
||||
Open threshold: {Math.round(vadAdvOpenRaw * 100)}%
|
||||
</span>
|
||||
<p>Minimum confidence required to open the gate.</p>
|
||||
<div className={styles.volumeSlider}>
|
||||
<label>Attack: {noiseGateAttackRaw} ms</label>
|
||||
<p>How quickly the gate opens when signal exceeds threshold.</p>
|
||||
<Slider
|
||||
label="VAD open threshold"
|
||||
value={vadAdvOpenRaw}
|
||||
onValueChange={setVadAdvOpenRaw}
|
||||
onValueCommit={setVadAdvOpen}
|
||||
min={0.1}
|
||||
max={0.95}
|
||||
step={0.05}
|
||||
|
||||
label="Noise gate attack"
|
||||
value={noiseGateAttackRaw}
|
||||
onValueChange={setNoiseGateAttackRaw}
|
||||
onValueCommit={setNoiseGateAttack}
|
||||
min={1}
|
||||
max={100}
|
||||
step={1}
|
||||
tooltip={false}
|
||||
/>
|
||||
</div>
|
||||
<div className={styles.volumeSlider}>
|
||||
<span className={styles.sliderLabel}>
|
||||
Close threshold: {Math.round(vadAdvCloseRaw * 100)}%
|
||||
</span>
|
||||
<p>Probability must drop below this to start the hold/close sequence.</p>
|
||||
<label>Hold: {noiseGateHoldRaw} ms</label>
|
||||
<p>How long the gate stays open after signal drops below threshold.</p>
|
||||
<Slider
|
||||
label="VAD close threshold"
|
||||
value={vadAdvCloseRaw}
|
||||
onValueChange={setVadAdvCloseRaw}
|
||||
onValueCommit={setVadAdvClose}
|
||||
min={0.05}
|
||||
max={0.9}
|
||||
step={0.05}
|
||||
|
||||
label="Noise gate hold"
|
||||
value={noiseGateHoldRaw}
|
||||
onValueChange={setNoiseGateHoldRaw}
|
||||
onValueCommit={setNoiseGateHold}
|
||||
min={0}
|
||||
max={500}
|
||||
step={10}
|
||||
tooltip={false}
|
||||
/>
|
||||
</div>
|
||||
<div className={`${styles.volumeSlider} ${styles.vadSpacedSlider}`}>
|
||||
<span className={styles.sliderLabel}>
|
||||
Hold time: {vadHoldRaw} ms
|
||||
</span>
|
||||
<p>How long to keep the gate open after speech drops below the close threshold.</p>
|
||||
<div className={styles.volumeSlider}>
|
||||
<label>Release: {noiseGateReleaseRaw} ms</label>
|
||||
<p>How quickly the gate closes after hold expires.</p>
|
||||
<Slider
|
||||
label="VAD hold time"
|
||||
value={vadHoldRaw}
|
||||
onValueChange={setVadHoldRaw}
|
||||
onValueCommit={setVadHold}
|
||||
min={0}
|
||||
max={2000}
|
||||
step={50}
|
||||
|
||||
label="Noise gate release"
|
||||
value={noiseGateReleaseRaw}
|
||||
onValueChange={setNoiseGateReleaseRaw}
|
||||
onValueCommit={setNoiseGateRelease}
|
||||
min={10}
|
||||
max={500}
|
||||
step={10}
|
||||
tooltip={false}
|
||||
/>
|
||||
</div>
|
||||
<div className={styles.restoreDefaults}>
|
||||
<Button
|
||||
kind="secondary"
|
||||
size="sm"
|
||||
onClick={(): void => {
|
||||
const defOpen = vadAdvancedOpenThresholdSetting.defaultValue;
|
||||
const defClose = vadAdvancedCloseThresholdSetting.defaultValue;
|
||||
const defHold = vadHoldTimeSetting.defaultValue;
|
||||
setVadAdvOpen(defOpen); setVadAdvOpenRaw(defOpen);
|
||||
setVadAdvClose(defClose); setVadAdvCloseRaw(defClose);
|
||||
setVadHold(defHold); setVadHoldRaw(defHold);
|
||||
setVadModeValue("standard");
|
||||
}}
|
||||
>
|
||||
<Button kind="secondary" size="sm" onClick={resetGateDefaults}>
|
||||
Restore defaults
|
||||
</Button>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
<div className={styles.noiseGateSection}>
|
||||
<Heading
|
||||
type="body"
|
||||
weight="semibold"
|
||||
size="sm"
|
||||
as="h4"
|
||||
className={styles.noiseGateHeading}
|
||||
>
|
||||
Voice Activity Detection
|
||||
</Heading>
|
||||
<Separator className={styles.noiseGateSeparator} />
|
||||
<FieldRow>
|
||||
<InputField
|
||||
id="vadEnabled"
|
||||
type="checkbox"
|
||||
label="Enable voice activity detection"
|
||||
description="Uses TEN-VAD to mute audio when no speech is detected (~10 ms latency)."
|
||||
checked={vadActive}
|
||||
onChange={(e: ChangeEvent<HTMLInputElement>): void =>
|
||||
setVadActive(e.target.checked)
|
||||
}
|
||||
/>
|
||||
</FieldRow>
|
||||
{vadActive && (
|
||||
<>
|
||||
<Form style={{ marginTop: "-16px" }}>
|
||||
<InlineField
|
||||
name={vadModeRadioGroup}
|
||||
control={
|
||||
<RadioControl
|
||||
checked={vadModeValue === "standard"}
|
||||
value="standard"
|
||||
onChange={(): void => setVadModeValue("standard")}
|
||||
/>
|
||||
}
|
||||
>
|
||||
<Label>Standard</Label>
|
||||
<HelpMessage>256 samples / 16 ms — comfortable feel.</HelpMessage>
|
||||
</InlineField>
|
||||
<InlineField
|
||||
name={vadModeRadioGroup}
|
||||
control={
|
||||
<RadioControl
|
||||
checked={vadModeValue === "aggressive"}
|
||||
value="aggressive"
|
||||
onChange={(): void => setVadModeValue("aggressive")}
|
||||
/>
|
||||
}
|
||||
>
|
||||
<Label>Aggressive</Label>
|
||||
<HelpMessage>160 samples / 10 ms — lowest possible latency.</HelpMessage>
|
||||
</InlineField>
|
||||
</Form>
|
||||
<div style={{ marginTop: "16px" }} className={`${styles.volumeSlider} ${styles.thresholdSlider}`}>
|
||||
<span className={styles.sliderLabel}>Open threshold: {Math.round(vadPositiveThresholdRaw * 100)}%</span>
|
||||
<p>How confident the model must be before opening the gate.</p>
|
||||
<Slider
|
||||
label="VAD open threshold"
|
||||
value={vadPositiveThresholdRaw}
|
||||
onValueChange={setVadPositiveThresholdRaw}
|
||||
onValueCommit={setVadPositiveThreshold}
|
||||
min={0.1}
|
||||
max={0.9}
|
||||
step={0.05}
|
||||
tooltip={false}
|
||||
/>
|
||||
</div>
|
||||
<div className={styles.volumeSlider}>
|
||||
<span className={styles.sliderLabel}>Close threshold: {Math.round(vadNegativeThresholdRaw * 100)}%</span>
|
||||
<p>How low the probability must drop before closing the gate.</p>
|
||||
<Slider
|
||||
label="VAD close threshold"
|
||||
value={vadNegativeThresholdRaw}
|
||||
onValueChange={setVadNegativeThresholdRaw}
|
||||
onValueCommit={setVadNegativeThreshold}
|
||||
min={0.05}
|
||||
max={0.7}
|
||||
step={0.05}
|
||||
tooltip={false}
|
||||
/>
|
||||
</div>
|
||||
<div className={styles.restoreDefaults}>
|
||||
<Button
|
||||
kind="secondary"
|
||||
size="sm"
|
||||
onClick={(): void => {
|
||||
const pos = vadPositiveThresholdSetting.defaultValue;
|
||||
const neg = vadNegativeThresholdSetting.defaultValue;
|
||||
setVadPositiveThreshold(pos); setVadPositiveThresholdRaw(pos);
|
||||
setVadNegativeThreshold(neg); setVadNegativeThresholdRaw(neg);
|
||||
}}
|
||||
>
|
||||
Restore defaults
|
||||
</Button>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
<div className={styles.noiseGateSection}>
|
||||
<Heading
|
||||
type="body"
|
||||
weight="semibold"
|
||||
size="sm"
|
||||
as="h4"
|
||||
className={styles.noiseGateHeading}
|
||||
>
|
||||
Transient Suppressor
|
||||
</Heading>
|
||||
<Separator className={styles.noiseGateSeparator} />
|
||||
<FieldRow>
|
||||
<InputField
|
||||
id="transientEnabled"
|
||||
type="checkbox"
|
||||
label="Enable transient suppressor"
|
||||
description="Cut sudden loud impacts like desk hits or mic bumps."
|
||||
checked={transientEnabled}
|
||||
onChange={(e: ChangeEvent<HTMLInputElement>): void =>
|
||||
setTransientEnabled(e.target.checked)
|
||||
}
|
||||
/>
|
||||
</FieldRow>
|
||||
{transientEnabled && (
|
||||
<>
|
||||
<div className={`${styles.volumeSlider} ${styles.thresholdSlider}`}>
|
||||
<span className={styles.sliderLabel}>Sensitivity: {transientThresholdRaw} dB above background</span>
|
||||
<p>Lower values catch more impacts; higher values only catch the loudest ones.</p>
|
||||
<Slider
|
||||
label="Transient threshold"
|
||||
value={transientThresholdRaw}
|
||||
onValueChange={setTransientThresholdRaw}
|
||||
onValueCommit={setTransientThreshold}
|
||||
min={8}
|
||||
max={30}
|
||||
step={1}
|
||||
tooltip={false}
|
||||
/>
|
||||
</div>
|
||||
<div className={styles.volumeSlider}>
|
||||
<span className={styles.sliderLabel}>Release: {transientReleaseRaw} ms</span>
|
||||
<p>How quickly audio returns after suppression.</p>
|
||||
<Slider
|
||||
label="Transient release"
|
||||
value={transientReleaseRaw}
|
||||
onValueChange={setTransientReleaseRaw}
|
||||
onValueCommit={setTransientRelease}
|
||||
min={20}
|
||||
max={200}
|
||||
step={10}
|
||||
tooltip={false}
|
||||
/>
|
||||
</div>
|
||||
<div className={styles.restoreDefaults}>
|
||||
<Button kind="secondary" size="sm" onClick={resetTransientDefaults}>
|
||||
Restore defaults
|
||||
</Button>
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
|
||||
@@ -129,16 +129,44 @@ export const alwaysShowIphoneEarpiece = new Setting<boolean>(
|
||||
false,
|
||||
);
|
||||
|
||||
export const noiseGateEnabled = new Setting<boolean>(
|
||||
"noise-gate-enabled",
|
||||
false,
|
||||
);
|
||||
// Threshold in dBFS — gate opens above this, closes below it
|
||||
export const noiseGateThreshold = new Setting<number>(
|
||||
"noise-gate-threshold",
|
||||
-60,
|
||||
);
|
||||
// Time in ms for the gate to fully open after signal exceeds threshold
|
||||
export const noiseGateAttack = new Setting<number>("noise-gate-attack", 25);
|
||||
// Time in ms the gate stays open after signal drops below threshold
|
||||
export const noiseGateHold = new Setting<number>("noise-gate-hold", 200);
|
||||
// Time in ms for the gate to fully close after hold expires
|
||||
export const noiseGateRelease = new Setting<number>("noise-gate-release", 150);
|
||||
|
||||
export const vadEnabled = new Setting<boolean>("vad-enabled", false);
|
||||
// Simple mode: single sensitivity slider (open threshold); close = open - 0.1
|
||||
export const vadPositiveThreshold = new Setting<number>("vad-positive-threshold", 0.7);
|
||||
// standard: 5ms/20ms aggressive: 1ms/5ms loose: 12ms/32ms
|
||||
export const vadMode = new Setting<"standard" | "aggressive" | "loose">("vad-mode", "standard");
|
||||
// Advanced settings (override simple mode when enabled)
|
||||
export const vadAdvancedEnabled = new Setting<boolean>("vad-advanced-enabled", false);
|
||||
export const vadAdvancedOpenThreshold = new Setting<number>("vad-advanced-open-threshold", 0.7);
|
||||
export const vadAdvancedCloseThreshold = new Setting<number>("vad-advanced-close-threshold", 0.6);
|
||||
export const vadHoldTime = new Setting<number>("vad-hold-time", 300);
|
||||
// Probability above which the VAD opens the gate (0–1)
|
||||
export const vadPositiveThreshold = new Setting<number>("vad-positive-threshold", 0.2);
|
||||
// Probability below which the VAD closes the gate (0–1)
|
||||
export const vadNegativeThreshold = new Setting<number>("vad-negative-threshold", 0.1);
|
||||
// standard: 5 ms open / 20 ms close ramp aggressive: 1 ms open / 5 ms close ramp
|
||||
export const vadMode = new Setting<"standard" | "aggressive">("vad-mode", "standard");
|
||||
|
||||
export const transientSuppressorEnabled = new Setting<boolean>(
|
||||
"transient-suppressor-enabled",
|
||||
false,
|
||||
);
|
||||
// How many dB above the background RMS a peak must be to trigger suppression
|
||||
export const transientThreshold = new Setting<number>(
|
||||
"transient-suppressor-threshold",
|
||||
15,
|
||||
);
|
||||
// Time in ms for suppression to fade after transient ends
|
||||
export const transientRelease = new Setting<number>(
|
||||
"transient-suppressor-release",
|
||||
80,
|
||||
);
|
||||
|
||||
export enum MatrixRTCMode {
|
||||
Legacy = "legacy",
|
||||
|
||||
@@ -33,18 +33,23 @@ import {
|
||||
} from "../../../livekit/TrackProcessorContext.tsx";
|
||||
import { getUrlParams } from "../../../UrlParams.ts";
|
||||
import {
|
||||
noiseGateEnabled,
|
||||
noiseGateThreshold,
|
||||
noiseGateAttack,
|
||||
noiseGateHold,
|
||||
noiseGateRelease,
|
||||
transientSuppressorEnabled,
|
||||
transientThreshold,
|
||||
transientRelease,
|
||||
vadEnabled,
|
||||
vadPositiveThreshold,
|
||||
vadNegativeThreshold,
|
||||
vadMode,
|
||||
vadAdvancedEnabled,
|
||||
vadAdvancedOpenThreshold,
|
||||
vadAdvancedCloseThreshold,
|
||||
vadHoldTime,
|
||||
} from "../../../settings/settings.ts";
|
||||
import {
|
||||
type TenVadParams,
|
||||
TenVadTransformer,
|
||||
} from "../../../livekit/TenVadTransformer.ts";
|
||||
type NoiseGateParams,
|
||||
NoiseGateTransformer,
|
||||
} from "../../../livekit/NoiseGateTransformer.ts";
|
||||
import { observeTrackReference$ } from "../../observeTrackReference";
|
||||
import { type Connection } from "../remoteMembers/Connection.ts";
|
||||
import { ObservableScope } from "../../ObservableScope.ts";
|
||||
@@ -89,7 +94,7 @@ export class Publisher {
|
||||
// Setup track processor syncing (blur)
|
||||
this.observeTrackProcessors(this.scope, room, trackerProcessorState$);
|
||||
// Setup noise gate on the local microphone track
|
||||
this.applyTenVad(this.scope, room);
|
||||
this.applyNoiseGate(this.scope, room);
|
||||
// Observe media device changes and update LiveKit active devices accordingly
|
||||
this.observeMediaDevices(this.scope, devices, controlledAudioDevices);
|
||||
|
||||
@@ -417,7 +422,7 @@ export class Publisher {
|
||||
});
|
||||
}
|
||||
|
||||
private applyTenVad(scope: ObservableScope, room: LivekitRoom): void {
|
||||
private applyNoiseGate(scope: ObservableScope, room: LivekitRoom): void {
|
||||
// Observe the local microphone track
|
||||
const audioTrack$ = scope.behavior(
|
||||
observeTrackReference$(
|
||||
@@ -432,55 +437,49 @@ export class Publisher {
|
||||
null,
|
||||
);
|
||||
|
||||
let transformer: TenVadTransformer | null = null;
|
||||
let transformer: NoiseGateTransformer | null = null;
|
||||
let audioCtx: AudioContext | null = null;
|
||||
|
||||
const currentParams = (): TenVadParams => {
|
||||
const isAdvanced = vadAdvancedEnabled.getValue();
|
||||
if (isAdvanced) {
|
||||
return {
|
||||
vadEnabled: vadEnabled.getValue(),
|
||||
vadPositiveThreshold: vadAdvancedOpenThreshold.getValue(),
|
||||
vadNegativeThreshold: vadAdvancedCloseThreshold.getValue(),
|
||||
vadMode: vadMode.getValue(),
|
||||
holdMs: vadHoldTime.getValue(),
|
||||
};
|
||||
}
|
||||
const openT = vadPositiveThreshold.getValue();
|
||||
return {
|
||||
vadEnabled: vadEnabled.getValue(),
|
||||
vadPositiveThreshold: openT,
|
||||
vadNegativeThreshold: Math.max(0, openT - 0.1),
|
||||
vadMode: "standard",
|
||||
holdMs: 0,
|
||||
};
|
||||
};
|
||||
const currentParams = (): NoiseGateParams => ({
|
||||
noiseGateActive: noiseGateEnabled.getValue(),
|
||||
threshold: noiseGateThreshold.getValue(),
|
||||
attackMs: noiseGateAttack.getValue(),
|
||||
holdMs: noiseGateHold.getValue(),
|
||||
releaseMs: noiseGateRelease.getValue(),
|
||||
transientEnabled: transientSuppressorEnabled.getValue(),
|
||||
transientThresholdDb: transientThreshold.getValue(),
|
||||
transientReleaseMs: transientRelease.getValue(),
|
||||
vadEnabled: vadEnabled.getValue(),
|
||||
vadPositiveThreshold: vadPositiveThreshold.getValue(),
|
||||
vadNegativeThreshold: vadNegativeThreshold.getValue(),
|
||||
vadAggressive: vadMode.getValue() === "aggressive",
|
||||
});
|
||||
|
||||
// Attach / detach processor when VAD is toggled or the track changes.
|
||||
combineLatest([audioTrack$, vadEnabled.value$])
|
||||
// Attach / detach processor when any processing feature changes or the track changes.
|
||||
combineLatest([audioTrack$, noiseGateEnabled.value$, vadEnabled.value$, transientSuppressorEnabled.value$])
|
||||
.pipe(scope.bind())
|
||||
.subscribe(([audioTrack, vadActive]) => {
|
||||
.subscribe(([audioTrack, ngEnabled, vadActive, transientActive]) => {
|
||||
if (!audioTrack) return;
|
||||
const shouldAttach = vadActive;
|
||||
const shouldAttach = ngEnabled || vadActive || transientActive;
|
||||
if (shouldAttach && !audioTrack.getProcessor()) {
|
||||
const params = currentParams();
|
||||
this.logger.info("[TenVad] attaching processor, params:", params);
|
||||
transformer = new TenVadTransformer(params);
|
||||
this.logger.info("[NoiseGate] attaching processor, params:", params);
|
||||
transformer = new NoiseGateTransformer(params);
|
||||
audioCtx = new AudioContext();
|
||||
this.logger.info("[TenVad] AudioContext state before resume:", audioCtx.state);
|
||||
this.logger.info("[NoiseGate] AudioContext state before resume:", audioCtx.state);
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(audioTrack as any).setAudioContext(audioCtx);
|
||||
audioCtx.resume().then(async () => {
|
||||
this.logger.info("[TenVad] AudioContext state after resume:", audioCtx?.state);
|
||||
this.logger.info("[NoiseGate] AudioContext state after resume:", audioCtx?.state);
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
return audioTrack.setProcessor(transformer as any);
|
||||
}).then(() => {
|
||||
this.logger.info("[TenVad] setProcessor resolved");
|
||||
this.logger.info("[NoiseGate] setProcessor resolved");
|
||||
}).catch((e: unknown) => {
|
||||
this.logger.error("[TenVad] setProcessor failed", e);
|
||||
this.logger.error("[NoiseGate] setProcessor failed", e);
|
||||
});
|
||||
} else if (!shouldAttach && audioTrack.getProcessor()) {
|
||||
this.logger.info("[TenVad] removing processor");
|
||||
this.logger.info("[NoiseGate] removing processor");
|
||||
void audioTrack.stopProcessor();
|
||||
void audioCtx?.close();
|
||||
audioCtx = null;
|
||||
@@ -488,29 +487,46 @@ export class Publisher {
|
||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||
(audioTrack as any).setAudioContext(undefined);
|
||||
} else if (shouldAttach && audioTrack.getProcessor()) {
|
||||
// Processor already attached — push updated params (e.g. vadActive toggled)
|
||||
// Processor already attached — push updated params (e.g. noiseGateActive toggled)
|
||||
transformer?.updateParams(currentParams());
|
||||
} else {
|
||||
this.logger.info(
|
||||
"[TenVad] tick — vadActive:", vadActive,
|
||||
"[NoiseGate] tick — ngEnabled:", ngEnabled,
|
||||
"vadActive:", vadActive,
|
||||
"hasProcessor:", !!audioTrack.getProcessor(),
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
// Push VAD param changes to the live worklet.
|
||||
// Push all param changes (noise gate + VAD) to the live worklet.
|
||||
combineLatest([
|
||||
noiseGateEnabled.value$,
|
||||
noiseGateThreshold.value$,
|
||||
noiseGateAttack.value$,
|
||||
noiseGateHold.value$,
|
||||
noiseGateRelease.value$,
|
||||
transientSuppressorEnabled.value$,
|
||||
transientThreshold.value$,
|
||||
transientRelease.value$,
|
||||
vadEnabled.value$,
|
||||
vadPositiveThreshold.value$,
|
||||
vadNegativeThreshold.value$,
|
||||
vadMode.value$,
|
||||
vadAdvancedEnabled.value$,
|
||||
vadAdvancedOpenThreshold.value$,
|
||||
vadAdvancedCloseThreshold.value$,
|
||||
vadHoldTime.value$,
|
||||
])
|
||||
.pipe(scope.bind())
|
||||
.subscribe(() => {
|
||||
transformer?.updateParams(currentParams());
|
||||
.subscribe(([
|
||||
noiseGateActive, threshold, attackMs, holdMs, releaseMs,
|
||||
transientEnabled, transientThresholdDb, transientReleaseMs,
|
||||
vadActive, vadPos, vadNeg, vadModeValue,
|
||||
]) => {
|
||||
transformer?.updateParams({
|
||||
noiseGateActive, threshold, attackMs, holdMs, releaseMs,
|
||||
transientEnabled, transientThresholdDb, transientReleaseMs,
|
||||
vadEnabled: vadActive,
|
||||
vadPositiveThreshold: vadPos,
|
||||
vadNegativeThreshold: vadNeg,
|
||||
vadAggressive: vadModeValue === "aggressive",
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@@ -530,4 +546,5 @@ export class Publisher {
|
||||
);
|
||||
trackProcessorSync(scope, track$, trackerProcessorState$);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -7,7 +7,6 @@ Please see LICENSE in the repository root for full details.
|
||||
|
||||
import {
|
||||
loadEnv,
|
||||
PluginOption,
|
||||
searchForWorkspaceRoot,
|
||||
type ConfigEnv,
|
||||
type UserConfig,
|
||||
@@ -34,7 +33,8 @@ export default ({
|
||||
// In future we might be able to do what is needed via code splitting at
|
||||
// build time.
|
||||
process.env.VITE_PACKAGE = packageType ?? "full";
|
||||
const plugins: PluginOption[] = [
|
||||
|
||||
const plugins = [
|
||||
react(),
|
||||
svgrPlugin({
|
||||
svgrOptions: {
|
||||
|
||||
Reference in New Issue
Block a user