1 Commits

Author SHA1 Message Date
mk
92f0cf2785 feat: adding ten-vad wasm to project, configuration options on audio
Some checks failed
Build / build_full_element_call (pull_request) Has been cancelled
Build / build_embedded_element_call (pull_request) Has been cancelled
Build / build_sdk_element_call (pull_request) Has been cancelled
Lint, format & type check / Lint, format & type check (pull_request) Has been cancelled
Build & publish embedded packages for releases / Versioning (pull_request) Has been cancelled
Test / Run unit tests (pull_request) Has been cancelled
Test / Run end-to-end tests (pull_request) Has been cancelled
GitHub Actions Security Analysis with zizmor 🌈 / Run zizmor 🌈 (pull_request) Has been cancelled
Prevent blocked / Prevent blocked (pull_request_target) Has been cancelled
Build / deploy_develop (pull_request) Has been cancelled
Build / docker_for_develop (pull_request) Has been cancelled
Build & publish embedded packages for releases / build_element_call (pull_request) Has been cancelled
Build & publish embedded packages for releases / Publish tarball (pull_request) Has been cancelled
Build & publish embedded packages for releases / Publish NPM (pull_request) Has been cancelled
Build & publish embedded packages for releases / Publish Android AAR (pull_request) Has been cancelled
Build & publish embedded packages for releases / Publish SwiftPM Library (pull_request) Has been cancelled
Build & publish embedded packages for releases / Update release notes (pull_request) Has been cancelled
settings
2026-03-28 20:42:50 -03:00
11 changed files with 376 additions and 779 deletions

View File

@@ -31,7 +31,6 @@ interface Props {
max: number; max: number;
step: number; step: number;
disabled?: boolean; disabled?: boolean;
tooltip?: boolean;
} }
/** /**
@@ -47,7 +46,6 @@ export const Slider: FC<Props> = ({
max, max,
step, step,
disabled, disabled,
tooltip = true,
}) => { }) => {
const onValueChange = useCallback( const onValueChange = useCallback(
([v]: number[]) => onValueChangeProp(v), ([v]: number[]) => onValueChangeProp(v),
@@ -73,13 +71,9 @@ export const Slider: FC<Props> = ({
<Range className={styles.highlight} /> <Range className={styles.highlight} />
</Track> </Track>
{/* Note: This is expected not to be visible on mobile.*/} {/* Note: This is expected not to be visible on mobile.*/}
{tooltip ? ( <Tooltip placement="top" label={Math.round(value * 100).toString() + "%"}>
<Tooltip placement="top" label={Math.round(value * 100).toString() + "%"}>
<Thumb className={styles.handle} aria-label={label} />
</Tooltip>
) : (
<Thumb className={styles.handle} aria-label={label} /> <Thumb className={styles.handle} aria-label={label} />
)} </Tooltip>
</Root> </Root>
); );
}; };

View File

@@ -74,8 +74,6 @@ export function LivekitRoomAudioRenderer({
) )
// Only keep audio tracks // Only keep audio tracks
.filter((ref) => ref.publication.kind === Track.Kind.Audio) .filter((ref) => ref.publication.kind === Track.Kind.Audio)
// Never render local participant's own audio back to themselves
.filter((ref) => !ref.participant.isLocal)
// Only keep tracks from participants that are in the validIdentities list // Only keep tracks from participants that are in the validIdentities list
.filter((ref) => { .filter((ref) => {
const isValid = validIdentities.includes(ref.participant.identity); const isValid = validIdentities.includes(ref.participant.identity);

View File

@@ -23,19 +23,13 @@ declare function registerProcessor(
processorCtor: new (...args: any[]) => AudioWorkletProcessor, processorCtor: new (...args: any[]) => AudioWorkletProcessor,
): void; ): void;
interface NoiseGateParams { interface TenVadParams {
noiseGateActive: boolean;
threshold: number; // dBFS — gate opens above this, closes below it
attackMs: number;
holdMs: number;
releaseMs: number;
transientEnabled: boolean;
transientThresholdDb: number; // dB above background RMS that triggers suppression
transientReleaseMs: number; // how quickly suppression fades after transient ends
// TEN-VAD params // TEN-VAD params
vadEnabled: boolean; vadEnabled: boolean;
vadPositiveThreshold: number; // open gate when prob >= this (01) vadPositiveThreshold: number; // open gate when prob >= this (01)
vadNegativeThreshold: number; // close gate when prob < this (01) vadNegativeThreshold: number; // close gate when prob < this (01)
vadMode: "standard" | "aggressive" | "loose";
holdMs: number; // hold time before closing gate (ms); 0 = no hold
} }
interface VADGateMessage { interface VADGateMessage {
@@ -43,10 +37,6 @@ interface VADGateMessage {
open: boolean; open: boolean;
} }
function dbToLinear(db: number): number {
return Math.pow(10, db / 20);
}
/** /**
* Thin synchronous wrapper around the TEN-VAD Emscripten WASM module. * Thin synchronous wrapper around the TEN-VAD Emscripten WASM module.
* Instantiated synchronously in the AudioWorklet constructor from a * Instantiated synchronously in the AudioWorklet constructor from a
@@ -170,55 +160,39 @@ class TenVADRuntime {
} }
/** /**
* AudioWorkletProcessor implementing a noise gate, an optional transient * AudioWorkletProcessor implementing an in-worklet TEN-VAD gate running
* suppressor, and an optional in-worklet TEN-VAD gate all running * per-sample.
* per-sample in a single pass.
*
* Noise gate: opens when instantaneous peak exceeds threshold, closes below.
* Attack, hold, and release times smooth the attenuation envelope.
*
* Transient suppressor: tracks a slow-moving RMS background level. When the
* instantaneous peak exceeds the background by more than transientThresholdDb,
* gain is instantly cut to 0 and releases over transientReleaseMs.
* *
* TEN-VAD gate: accumulates audio with 3:1 decimation (48 kHz 16 kHz), * TEN-VAD gate: accumulates audio with 3:1 decimation (48 kHz 16 kHz),
* runs the TEN-VAD model synchronously every 256 samples (16 ms), and * runs the TEN-VAD model synchronously every 256 samples (16 ms), and
* controls vadGateOpen with hysteresis. No IPC round-trip required. * controls vadGateOpen with hysteresis. No IPC round-trip required.
* Asymmetric ramp: 5 ms open (minimise speech onset masking), 20 ms close
* (de-click on silence).
*/ */
class NoiseGateProcessor extends AudioWorkletProcessor { class TenVadProcessor extends AudioWorkletProcessor {
// Noise gate state
private noiseGateActive = true;
private threshold = dbToLinear(-60);
private attackRate = 1.0 / (0.025 * sampleRate);
private releaseRate = 1.0 / (0.15 * sampleRate);
private holdTime = 0.2;
private isOpen = false;
private gateAttenuation = 0;
private heldTime = 0;
// Transient suppressor state
private transientEnabled = false;
private transientRatio = dbToLinear(15);
private transientReleaseRate = 1.0 / (0.08 * sampleRate);
private transientAttenuation = 1.0;
private slowRms = 0;
private rmsCoeff = Math.exp(-1.0 / (0.2 * sampleRate));
// VAD gate state // VAD gate state
private vadGateOpen = true; // starts open; TEN-VAD closes it on first silent frame private vadGateOpen = true; // starts open; TEN-VAD closes it on first silent frame
private vadAttenuation = 1.0; private vadAttenuation = 1.0;
private readonly vadRampRate = 1.0 / (0.02 * sampleRate); // Asymmetric ramp rates — recomputed in updateParams based on vadAggressive
private vadOpenRampRate = 1.0 / (0.005 * sampleRate); // default: 5 ms
private vadCloseRampRate = 1.0 / (0.02 * sampleRate); // default: 20 ms
// TEN-VAD state // TEN-VAD state
private vadEnabled = false; private vadEnabled = false;
private vadPositiveThreshold = 0.5; private vadPositiveThreshold = 0.5;
private vadNegativeThreshold = 0.3; private vadNegativeThreshold = 0.3;
private holdMs = 0;
private vadHoldHops = 0; // hold expressed in VAD hops
private vadHoldCounter = 0; // hops of continuous sub-threshold signal while gate is open
private tenVadRuntime: TenVADRuntime | null = null; private tenVadRuntime: TenVADRuntime | null = null;
private tenVadModule: WebAssembly.Module | undefined = undefined;
// 3:1 decimation from AudioContext sample rate to 16 kHz // 3:1 decimation from AudioContext sample rate to 16 kHz
private readonly decRatio = Math.max(1, Math.round(sampleRate / 16000)); private readonly decRatio = Math.max(1, Math.round(sampleRate / 16000));
private decPhase = 0; private decPhase = 0;
private decAcc = 0; private decAcc = 0;
// Buffer sized for max hop (256); vadHopSize tracks how many samples to collect
private readonly vadHopBuf = new Int16Array(256); private readonly vadHopBuf = new Int16Array(256);
private vadHopSize = 256; // standard: 256 (16 ms), aggressive: 160 (10 ms)
private vadHopCount = 0; private vadHopCount = 0;
private logCounter = 0; private logCounter = 0;
@@ -229,75 +203,95 @@ class NoiseGateProcessor extends AudioWorkletProcessor {
super(options); super(options);
// Try to instantiate TEN-VAD from the pre-compiled module passed by the main thread // Try to instantiate TEN-VAD from the pre-compiled module passed by the main thread
const tenVadModule = options?.processorOptions?.tenVadModule as this.tenVadModule = options?.processorOptions?.tenVadModule as
| WebAssembly.Module | WebAssembly.Module
| undefined; | undefined;
if (tenVadModule) { if (this.tenVadModule) {
try { try {
// hopSize = 256 samples @ 16 kHz = 16 ms; threshold = 0.5 (overridden via params) // Default: standard mode 256 samples @ 16 kHz = 16 ms
this.tenVadRuntime = new TenVADRuntime(tenVadModule, 256, 0.5); this.tenVadRuntime = new TenVADRuntime(this.tenVadModule, 256, 0.5);
this.port.postMessage({ this.port.postMessage({
type: "log", type: "log",
msg: "[NoiseGate worklet] TEN-VAD runtime initialized, decRatio=" + this.decRatio, msg: "[TenVad worklet] TEN-VAD runtime initialized, decRatio=" + this.decRatio,
}); });
} catch (e) { } catch (e) {
this.port.postMessage({ this.port.postMessage({
type: "log", type: "log",
msg: "[NoiseGate worklet] TEN-VAD init failed: " + String(e), msg: "[TenVad worklet] TEN-VAD init failed: " + String(e),
}); });
} }
} }
this.port.onmessage = ( this.port.onmessage = (
e: MessageEvent<NoiseGateParams | VADGateMessage>, e: MessageEvent<TenVadParams | VADGateMessage>,
): void => { ): void => {
if ((e.data as VADGateMessage).type === "vad-gate") { if ((e.data as VADGateMessage).type === "vad-gate") {
this.vadGateOpen = (e.data as VADGateMessage).open; this.vadGateOpen = (e.data as VADGateMessage).open;
} else { } else {
this.updateParams(e.data as NoiseGateParams); this.updateParams(e.data as TenVadParams);
} }
}; };
this.updateParams({ this.updateParams({
noiseGateActive: true,
threshold: -60,
attackMs: 25,
holdMs: 200,
releaseMs: 150,
transientEnabled: false,
transientThresholdDb: 15,
transientReleaseMs: 80,
vadEnabled: false, vadEnabled: false,
vadPositiveThreshold: 0.5, vadPositiveThreshold: 0.5,
vadNegativeThreshold: 0.3, vadNegativeThreshold: 0.3,
vadMode: "standard",
holdMs: 0,
}); });
this.port.postMessage({ this.port.postMessage({
type: "log", type: "log",
msg: "[NoiseGate worklet] constructor called, sampleRate=" + sampleRate, msg: "[TenVad worklet] constructor called, sampleRate=" + sampleRate,
}); });
} }
private updateParams(p: NoiseGateParams): void { private updateParams(p: TenVadParams): void {
this.noiseGateActive = p.noiseGateActive ?? true;
this.threshold = dbToLinear(p.threshold);
this.attackRate = 1.0 / ((p.attackMs / 1000) * sampleRate);
this.releaseRate = 1.0 / ((p.releaseMs / 1000) * sampleRate);
this.holdTime = p.holdMs / 1000;
this.transientEnabled = p.transientEnabled;
this.transientRatio = dbToLinear(p.transientThresholdDb);
this.transientReleaseRate = 1.0 / ((p.transientReleaseMs / 1000) * sampleRate);
this.vadEnabled = p.vadEnabled ?? false; this.vadEnabled = p.vadEnabled ?? false;
this.vadPositiveThreshold = p.vadPositiveThreshold ?? 0.5; this.vadPositiveThreshold = p.vadPositiveThreshold ?? 0.5;
this.vadNegativeThreshold = p.vadNegativeThreshold ?? 0.3; this.vadNegativeThreshold = p.vadNegativeThreshold ?? 0.3;
// When VAD is disabled, open the gate immediately this.holdMs = p.holdMs ?? 0;
const newMode = p.vadMode ?? "standard";
if (newMode === "aggressive") {
this.vadOpenRampRate = 1.0 / (0.001 * sampleRate); // 1 ms
this.vadCloseRampRate = 1.0 / (0.005 * sampleRate); // 5 ms
} else if (newMode === "loose") {
this.vadOpenRampRate = 1.0 / (0.012 * sampleRate); // 12 ms
this.vadCloseRampRate = 1.0 / (0.032 * sampleRate); // 32 ms
} else {
this.vadOpenRampRate = 1.0 / (0.005 * sampleRate); // 5 ms
this.vadCloseRampRate = 1.0 / (0.02 * sampleRate); // 20 ms
}
// Hop size: aggressive=160 (10 ms @ 16 kHz), others=256 (16 ms)
const newHopSize = newMode === "aggressive" ? 160 : 256;
if (newHopSize !== this.vadHopSize && this.tenVadModule) {
this.tenVadRuntime?.destroy();
this.tenVadRuntime = null;
this.vadHopCount = 0;
try {
this.tenVadRuntime = new TenVADRuntime(this.tenVadModule, newHopSize, 0.5);
} catch (e) {
this.port.postMessage({ type: "log", msg: "[TenVad worklet] TEN-VAD recreate failed: " + String(e) });
}
}
this.vadHopSize = newHopSize;
// Recompute hold in hops: ceil((holdMs / 1000) * 16000 / vadHopSize)
this.vadHoldHops = this.holdMs > 0
? Math.ceil((this.holdMs / 1000) * 16000 / this.vadHopSize)
: 0;
this.vadHoldCounter = 0;
if (!this.vadEnabled) this.vadGateOpen = true; if (!this.vadEnabled) this.vadGateOpen = true;
this.port.postMessage({ this.port.postMessage({
type: "log", type: "log",
msg: "[NoiseGate worklet] params updated: threshold=" + p.threshold msg: "[TenVad worklet] params updated: vadEnabled=" + p.vadEnabled
+ " vadEnabled=" + p.vadEnabled
+ " vadPos=" + p.vadPositiveThreshold + " vadPos=" + p.vadPositiveThreshold
+ " vadNeg=" + p.vadNegativeThreshold, + " vadNeg=" + p.vadNegativeThreshold
+ " vadMode=" + newMode
+ " holdMs=" + this.holdMs,
}); });
} }
@@ -306,63 +300,9 @@ class NoiseGateProcessor extends AudioWorkletProcessor {
const output = outputs[0]; const output = outputs[0];
if (!input || input.length === 0) return true; if (!input || input.length === 0) return true;
const channels = input.length;
const blockSize = input[0]?.length ?? 128; const blockSize = input[0]?.length ?? 128;
const samplePeriod = 1.0 / sampleRate;
for (let i = 0; i < blockSize; i++) { for (let i = 0; i < blockSize; i++) {
// Peak detection across all channels
let curLevel = Math.abs(input[0]?.[i] ?? 0);
for (let j = 1; j < channels; j++) {
curLevel = Math.max(curLevel, Math.abs(input[j]?.[i] ?? 0));
}
// --- Transient suppressor ---
let transientGain = 1.0;
if (this.transientEnabled) {
this.slowRms = Math.sqrt(
this.rmsCoeff * this.slowRms * this.slowRms +
(1.0 - this.rmsCoeff) * curLevel * curLevel,
);
const background = Math.max(this.slowRms, 1e-6);
if (curLevel > background * this.transientRatio) {
this.transientAttenuation = 0.0;
} else {
this.transientAttenuation = Math.min(
1.0,
this.transientAttenuation + this.transientReleaseRate,
);
}
transientGain = this.transientAttenuation;
}
// --- Noise gate ---
if (this.noiseGateActive) {
if (curLevel > this.threshold && !this.isOpen) {
this.isOpen = true;
}
if (curLevel <= this.threshold && this.isOpen) {
this.heldTime = 0;
this.isOpen = false;
}
if (this.isOpen) {
this.gateAttenuation = Math.min(
1.0,
this.gateAttenuation + this.attackRate,
);
} else {
this.heldTime += samplePeriod;
if (this.heldTime > this.holdTime) {
this.gateAttenuation = Math.max(
0.0,
this.gateAttenuation - this.releaseRate,
);
}
}
} else {
this.gateAttenuation = 1.0;
}
// --- TEN-VAD in-worklet processing --- // --- TEN-VAD in-worklet processing ---
// Accumulate raw mono samples with decRatio:1 decimation (48 kHz → 16 kHz). // Accumulate raw mono samples with decRatio:1 decimation (48 kHz → 16 kHz).
// Every 256 output samples (16 ms) run the WASM VAD and update vadGateOpen. // Every 256 output samples (16 ms) run the WASM VAD and update vadGateOpen.
@@ -382,33 +322,49 @@ class NoiseGateProcessor extends AudioWorkletProcessor {
: (avg * 32767 + 0.5) | 0; : (avg * 32767 + 0.5) | 0;
this.vadHopBuf[this.vadHopCount++] = s16; this.vadHopBuf[this.vadHopCount++] = s16;
if (this.vadHopCount >= 256) { if (this.vadHopCount >= this.vadHopSize) {
this.vadHopCount = 0; this.vadHopCount = 0;
const prob = this.tenVadRuntime.process(this.vadHopBuf); const prob = this.tenVadRuntime.process(this.vadHopBuf);
if (!this.vadGateOpen && prob >= this.vadPositiveThreshold) { if (prob >= this.vadPositiveThreshold) {
// Speech detected — open gate, reset hold counter
this.vadGateOpen = true; this.vadGateOpen = true;
} else if (this.vadGateOpen && prob < this.vadNegativeThreshold) { this.vadHoldCounter = 0;
this.vadGateOpen = false; } else if (prob < this.vadNegativeThreshold) {
if (this.vadGateOpen) {
if (this.vadHoldHops === 0) {
this.vadGateOpen = false;
} else {
this.vadHoldCounter++;
if (this.vadHoldCounter >= this.vadHoldHops) {
this.vadGateOpen = false;
this.vadHoldCounter = 0;
}
}
}
} else {
// Ambiguous zone — reset hold counter so hold only fires on sustained silence
this.vadHoldCounter = 0;
} }
} }
} }
} }
// Ramp VAD attenuation toward target to avoid clicks // Asymmetric ramp: fast open (5 ms) to minimise speech onset masking,
// slow close (20 ms) to de-click on silence transitions.
const vadTarget = this.vadGateOpen ? 1.0 : 0.0; const vadTarget = this.vadGateOpen ? 1.0 : 0.0;
if (this.vadAttenuation < vadTarget) { if (this.vadAttenuation < vadTarget) {
this.vadAttenuation = Math.min( this.vadAttenuation = Math.min(
vadTarget, vadTarget,
this.vadAttenuation + this.vadRampRate, this.vadAttenuation + this.vadOpenRampRate,
); );
} else if (this.vadAttenuation > vadTarget) { } else if (this.vadAttenuation > vadTarget) {
this.vadAttenuation = Math.max( this.vadAttenuation = Math.max(
vadTarget, vadTarget,
this.vadAttenuation - this.vadRampRate, this.vadAttenuation - this.vadCloseRampRate,
); );
} }
const gain = this.gateAttenuation * transientGain * this.vadAttenuation; const gain = this.vadAttenuation;
for (let c = 0; c < output.length; c++) { for (let c = 0; c < output.length; c++) {
const inCh = input[c] ?? input[0]; const inCh = input[c] ?? input[0];
@@ -423,10 +379,7 @@ class NoiseGateProcessor extends AudioWorkletProcessor {
if (this.logCounter % 375 === 0) { if (this.logCounter % 375 === 0) {
this.port.postMessage({ this.port.postMessage({
type: "log", type: "log",
msg: "[NoiseGate worklet] gateOpen=" + this.isOpen msg: "[TenVad worklet] vadOpen=" + this.vadGateOpen
+ " gateAtten=" + this.gateAttenuation.toFixed(3)
+ " transientAtten=" + this.transientAttenuation.toFixed(3)
+ " vadOpen=" + this.vadGateOpen
+ " vadAtten=" + this.vadAttenuation.toFixed(3), + " vadAtten=" + this.vadAttenuation.toFixed(3),
}); });
} }
@@ -435,4 +388,4 @@ class NoiseGateProcessor extends AudioWorkletProcessor {
} }
} }
registerProcessor("noise-gate-processor", NoiseGateProcessor); registerProcessor("ten-vad-processor", TenVadProcessor);

View File

@@ -7,22 +7,19 @@ Please see LICENSE in the repository root for full details.
import { type Track } from "livekit-client"; import { type Track } from "livekit-client";
import { logger } from "matrix-js-sdk/lib/logger"; import { logger } from "matrix-js-sdk/lib/logger";
// ?worker&url tells Vite to compile the TypeScript worklet and return its URL.
// Without this, Vite copies the .ts file verbatim and the browser rejects it.
import compiledWorkletUrl from "./TenVadProcessor.worklet.ts?worker&url";
const log = logger.getChild("[NoiseGateTransformer]"); const log = logger.getChild("[TenVadTransformer]");
export interface NoiseGateParams { export interface TenVadParams {
noiseGateActive: boolean;
threshold: number; // dBFS — gate opens above this, closes below it
attackMs: number;
holdMs: number;
releaseMs: number;
transientEnabled: boolean;
transientThresholdDb: number; // dB above background RMS that triggers suppression
transientReleaseMs: number; // ms for suppression to fade after transient ends
// TEN-VAD params — processed entirely inside the AudioWorklet // TEN-VAD params — processed entirely inside the AudioWorklet
vadEnabled: boolean; vadEnabled: boolean;
vadPositiveThreshold: number; // open gate when isSpeech prob >= this (01) vadPositiveThreshold: number; // open gate when prob >= this (01)
vadNegativeThreshold: number; // close gate when isSpeech prob < this (01) vadNegativeThreshold: number; // close gate when prob < this (01); computed by Publisher
vadMode: "standard" | "aggressive" | "loose";
holdMs: number; // hold time before closing gate (ms); 0 = no hold
} }
/** /**
@@ -68,8 +65,7 @@ function getTenVADModule(): Promise<WebAssembly.Module> {
} }
/** /**
* LiveKit audio track processor that applies a noise gate, optional transient * LiveKit audio track processor that applies TEN-VAD via AudioWorklet.
* suppressor, and optional TEN-VAD gate via AudioWorklet.
* *
* The TEN-VAD WASM module is fetched once, compiled, and passed to the worklet * The TEN-VAD WASM module is fetched once, compiled, and passed to the worklet
* via processorOptions so it runs synchronously inside the audio thread * via processorOptions so it runs synchronously inside the audio thread
@@ -78,16 +74,16 @@ function getTenVADModule(): Promise<WebAssembly.Module> {
* Audio graph: sourceNode workletNode destinationNode * Audio graph: sourceNode workletNode destinationNode
* processedTrack is destinationNode.stream.getAudioTracks()[0] * processedTrack is destinationNode.stream.getAudioTracks()[0]
*/ */
export class NoiseGateTransformer implements AudioTrackProcessor { export class TenVadTransformer implements AudioTrackProcessor {
public readonly name = "noise-gate"; public readonly name = "ten-vad";
public processedTrack?: MediaStreamTrack; public processedTrack?: MediaStreamTrack;
private workletNode?: AudioWorkletNode; private workletNode?: AudioWorkletNode;
private sourceNode?: MediaStreamAudioSourceNode; private sourceNode?: MediaStreamAudioSourceNode;
private destinationNode?: MediaStreamAudioDestinationNode; private destinationNode?: MediaStreamAudioDestinationNode;
private params: NoiseGateParams; private params: TenVadParams;
public constructor(params: NoiseGateParams) { public constructor(params: TenVadParams) {
this.params = { ...params }; this.params = { ...params };
} }
@@ -105,17 +101,13 @@ export class NoiseGateTransformer implements AudioTrackProcessor {
log.warn("TEN-VAD WASM module unavailable — VAD disabled:", e); log.warn("TEN-VAD WASM module unavailable — VAD disabled:", e);
} }
const workletUrl = new URL( log.info("loading worklet from:", compiledWorkletUrl);
"./NoiseGateProcessor.worklet.ts", await audioContext.audioWorklet.addModule(compiledWorkletUrl);
import.meta.url,
);
log.info("loading worklet from:", workletUrl.href);
await audioContext.audioWorklet.addModule(workletUrl);
log.info("worklet module loaded"); log.info("worklet module loaded");
this.workletNode = new AudioWorkletNode( this.workletNode = new AudioWorkletNode(
audioContext, audioContext,
"noise-gate-processor", "ten-vad-processor",
{ {
processorOptions: { processorOptions: {
tenVadModule, tenVadModule,
@@ -158,7 +150,7 @@ export class NoiseGateTransformer implements AudioTrackProcessor {
} }
/** Push updated gate/VAD parameters to the running worklet. */ /** Push updated gate/VAD parameters to the running worklet. */
public updateParams(params: NoiseGateParams): void { public updateParams(params: TenVadParams): void {
this.params = { ...params }; this.params = { ...params };
this.sendParams(); this.sendParams();
} }

View File

@@ -1,87 +0,0 @@
/*
Copyright 2026 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
.wrapper {
display: flex;
align-items: center;
position: relative;
width: 100%;
}
.slider {
display: flex;
align-items: center;
position: relative;
width: 100%;
}
.track {
flex-grow: 1;
border-radius: var(--cpd-radius-pill-effect);
height: var(--cpd-space-4x);
outline: var(--cpd-border-width-1) solid var(--cpd-color-border-interactive-primary);
outline-offset: calc(-1 * var(--cpd-border-width-1));
cursor: pointer;
position: relative;
overflow: hidden;
/* Base background */
background: var(--cpd-color-bg-subtle-primary);
}
/* Live mic level fill — driven by --mic-level CSS variable */
.track::before {
content: "";
position: absolute;
inset: 0;
width: var(--mic-level, 0%);
background: var(--cpd-color-gray-600, #808080);
opacity: 0.55;
border-radius: var(--cpd-radius-pill-effect);
transition: width 40ms linear;
pointer-events: none;
}
/* Green when mic level is at or above the threshold */
.track[data-active="true"]::before {
background: var(--cpd-color-green-900, #1a7f4b);
}
/* Threshold marker — driven by --threshold-pct CSS variable */
.track::after {
content: "";
position: absolute;
top: 0;
bottom: 0;
left: var(--threshold-pct, 50%);
width: 2px;
background: var(--cpd-color-text-primary);
opacity: 0.6;
pointer-events: none;
transform: translateX(-50%);
}
/* Hide the Radix Range highlight — we use our own visuals */
.range {
display: none;
}
.handle {
display: block;
block-size: var(--cpd-space-5x);
inline-size: var(--cpd-space-5x);
border-radius: var(--cpd-radius-pill-effect);
background: var(--cpd-color-bg-action-primary-rest);
box-shadow: 0 0 0 2px var(--cpd-color-bg-canvas-default);
cursor: pointer;
transition: background-color ease 0.15s;
z-index: 1;
}
.handle:focus-visible {
outline: 2px solid var(--cpd-color-border-focused);
outline-offset: 2px;
}

View File

@@ -1,139 +0,0 @@
/*
Copyright 2026 New Vector Ltd.
SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details.
*/
import { Root, Track, Range, Thumb } from "@radix-ui/react-slider";
import { type FC, useCallback, useEffect, useRef } from "react";
import styles from "./NoiseLevelSlider.module.css";
interface Props {
label: string;
value: number;
onValueChange: (value: number) => void;
onValueCommit?: (value: number) => void;
min: number;
max: number;
step: number;
}
/**
* Threshold slider that shows live microphone input level as a background fill,
* similar to Discord's input sensitivity control.
*
* The green fill represents your current mic volume in real time.
* Drag the handle to set the gate threshold — audio below it will be silenced.
*/
export const NoiseLevelSlider: FC<Props> = ({
label,
value,
onValueChange: onValueChangeProp,
onValueCommit: onValueCommitProp,
min,
max,
step,
}) => {
const trackRef = useRef<HTMLSpanElement>(null);
const animFrameRef = useRef<number>(0);
const analyserRef = useRef<AnalyserNode | null>(null);
const streamRef = useRef<MediaStream | null>(null);
const dataRef = useRef<Float32Array<ArrayBuffer> | null>(null);
const thresholdPctRef = useRef<number>(0);
// Start mic monitoring via AnalyserNode
useEffect(() => {
let ctx: AudioContext | null = null;
navigator.mediaDevices
.getUserMedia({ audio: true, video: false })
.then((stream) => {
streamRef.current = stream;
ctx = new AudioContext();
const source = ctx.createMediaStreamSource(stream);
const analyser = ctx.createAnalyser();
analyser.fftSize = 1024;
analyser.smoothingTimeConstant = 0.6;
source.connect(analyser);
analyserRef.current = analyser;
dataRef.current = new Float32Array(analyser.fftSize) as Float32Array<ArrayBuffer>;
})
.catch(() => {
// Mic not available — live level stays at 0
});
return (): void => {
cancelAnimationFrame(animFrameRef.current);
streamRef.current?.getTracks().forEach((t) => t.stop());
streamRef.current = null;
analyserRef.current = null;
void ctx?.close();
};
}, []);
// rAF loop — reads RMS level and updates CSS variable on the track element
useEffect(() => {
const tick = (): void => {
animFrameRef.current = requestAnimationFrame(tick);
const analyser = analyserRef.current;
const data = dataRef.current;
const track = trackRef.current;
if (!analyser || !data || !track) return;
analyser.getFloatTimeDomainData(data);
// Peak detection — matches the gate's own measurement method
let peak = 0;
for (const s of data) peak = Math.max(peak, Math.abs(s));
const dbfs = peak > 0 ? 20 * Math.log10(peak) : -Infinity;
// Map dBFS value to slider percentage (same scale as min/max)
const clampedDb = Math.max(min, Math.min(max, dbfs));
const levelPct = ((clampedDb - min) / (max - min)) * 100;
track.style.setProperty("--mic-level", `${levelPct.toFixed(1)}%`);
track.dataset.active = String(levelPct >= thresholdPctRef.current);
};
animFrameRef.current = requestAnimationFrame(tick);
return (): void => cancelAnimationFrame(animFrameRef.current);
}, [min, max]);
// Keep threshold marker in sync with slider value
useEffect(() => {
const track = trackRef.current;
if (!track) return;
const thresholdPct = ((value - min) / (max - min)) * 100;
thresholdPctRef.current = thresholdPct;
track.style.setProperty("--threshold-pct", `${thresholdPct.toFixed(1)}%`);
}, [value, min, max]);
const onValueChange = useCallback(
([v]: number[]) => onValueChangeProp(v),
[onValueChangeProp],
);
const onValueCommit = useCallback(
([v]: number[]) => onValueCommitProp?.(v),
[onValueCommitProp],
);
return (
<Root
className={styles.slider}
value={[value]}
onValueChange={onValueChange}
onValueCommit={onValueCommit}
min={min}
max={max}
step={step}
>
<Track className={styles.track} ref={trackRef}>
<Range className={styles.range} />
</Track>
<Thumb className={styles.handle} aria-label={label} />
</Root>
);
};

View File

@@ -35,37 +35,30 @@ Please see LICENSE in the repository root for full details.
color: var(--cpd-color-text-secondary); color: var(--cpd-color-text-secondary);
} }
.noiseGateSection { .vadSection {
margin-block-start: var(--cpd-space-6x); margin-block-start: var(--cpd-space-6x);
} }
.noiseGateHeading { .vadHeading {
color: var(--cpd-color-text-secondary); color: var(--cpd-color-text-secondary);
margin-block: var(--cpd-space-3x) 0; margin-block: var(--cpd-space-3x) 0;
} }
.thresholdSlider { .vadSeparator {
margin-block-start: calc(-32px + var(--cpd-space-2x));
}
.noiseGateSeparator {
margin-block: 6px var(--cpd-space-4x); margin-block: 6px var(--cpd-space-4x);
} }
.advancedGate { .vadRampLabel {
margin-top: var(--cpd-space-3x); display: block;
margin-block: var(--cpd-space-6x) var(--cpd-space-1x);
} }
.advancedGateToggle { .vadRampForm {
all: unset; margin-top: 0;
cursor: pointer;
font: var(--cpd-font-body-sm-semibold);
color: var(--cpd-color-text-secondary);
user-select: none;
} }
.advancedGateToggle:hover { .vadSpacedSlider {
color: var(--cpd-color-text-primary); margin-block-start: var(--cpd-space-6x);
} }
.restoreDefaults { .restoreDefaults {

View File

@@ -5,10 +5,19 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial
Please see LICENSE in the repository root for full details. Please see LICENSE in the repository root for full details.
*/ */
import { type ChangeEvent, type FC, type ReactNode, useEffect, useState, useCallback } from "react"; import { type FC, type ReactNode, useEffect, useId, useState } from "react";
import { useTranslation } from "react-i18next"; import { useTranslation } from "react-i18next";
import { type MatrixClient } from "matrix-js-sdk"; import { type MatrixClient } from "matrix-js-sdk";
import { Button, Heading, Root as Form, Separator } from "@vector-im/compound-web"; import {
Button,
Heading,
HelpMessage,
InlineField,
Label,
RadioControl,
Root as Form,
Separator,
} from "@vector-im/compound-web";
import { type Room as LivekitRoom } from "livekit-client"; import { type Room as LivekitRoom } from "livekit-client";
import { Modal } from "../Modal"; import { Modal } from "../Modal";
@@ -24,21 +33,16 @@ import {
soundEffectVolume as soundEffectVolumeSetting, soundEffectVolume as soundEffectVolumeSetting,
backgroundBlur as backgroundBlurSetting, backgroundBlur as backgroundBlurSetting,
developerMode, developerMode,
noiseGateEnabled as noiseGateEnabledSetting,
noiseGateThreshold as noiseGateThresholdSetting,
noiseGateAttack as noiseGateAttackSetting,
noiseGateHold as noiseGateHoldSetting,
noiseGateRelease as noiseGateReleaseSetting,
transientSuppressorEnabled as transientSuppressorEnabledSetting,
transientThreshold as transientThresholdSetting,
transientRelease as transientReleaseSetting,
vadEnabled as vadEnabledSetting, vadEnabled as vadEnabledSetting,
vadPositiveThreshold as vadPositiveThresholdSetting, vadPositiveThreshold as vadPositiveThresholdSetting,
vadNegativeThreshold as vadNegativeThresholdSetting, vadMode as vadModeSetting,
vadAdvancedEnabled as vadAdvancedEnabledSetting,
vadAdvancedOpenThreshold as vadAdvancedOpenThresholdSetting,
vadAdvancedCloseThreshold as vadAdvancedCloseThresholdSetting,
vadHoldTime as vadHoldTimeSetting,
} from "./settings"; } from "./settings";
import { PreferencesSettingsTab } from "./PreferencesSettingsTab"; import { PreferencesSettingsTab } from "./PreferencesSettingsTab";
import { Slider } from "../Slider"; import { Slider } from "../Slider";
import { NoiseLevelSlider } from "./NoiseLevelSlider";
import { DeviceSelection } from "./DeviceSelection"; import { DeviceSelection } from "./DeviceSelection";
import { useTrackProcessor } from "../livekit/TrackProcessorContext"; import { useTrackProcessor } from "../livekit/TrackProcessorContext";
import { DeveloperSettingsTab } from "./DeveloperSettingsTab"; import { DeveloperSettingsTab } from "./DeveloperSettingsTab";
@@ -119,48 +123,25 @@ export const SettingsModal: FC<Props> = ({
const [soundVolumeRaw, setSoundVolumeRaw] = useState(soundVolume); const [soundVolumeRaw, setSoundVolumeRaw] = useState(soundVolume);
const [showDeveloperSettingsTab] = useSetting(developerMode); const [showDeveloperSettingsTab] = useSetting(developerMode);
// Noise gate settings
const [noiseGateEnabled, setNoiseGateEnabled] = useSetting(noiseGateEnabledSetting);
const [noiseGateThreshold, setNoiseGateThreshold] = useSetting(noiseGateThresholdSetting);
const [noiseGateThresholdRaw, setNoiseGateThresholdRaw] = useState(noiseGateThreshold);
const [noiseGateAttack, setNoiseGateAttack] = useSetting(noiseGateAttackSetting);
const [noiseGateAttackRaw, setNoiseGateAttackRaw] = useState(noiseGateAttack);
const [noiseGateHold, setNoiseGateHold] = useSetting(noiseGateHoldSetting);
const [noiseGateHoldRaw, setNoiseGateHoldRaw] = useState(noiseGateHold);
const [noiseGateRelease, setNoiseGateRelease] = useSetting(noiseGateReleaseSetting);
const [noiseGateReleaseRaw, setNoiseGateReleaseRaw] = useState(noiseGateRelease);
const [showAdvancedGate, setShowAdvancedGate] = useState(false);
// Voice activity detection // Voice activity detection
const vadStateGroup = useId();
const vadModeRadioGroup = useId();
const [vadActive, setVadActive] = useSetting(vadEnabledSetting); const [vadActive, setVadActive] = useSetting(vadEnabledSetting);
const [vadPositiveThreshold, setVadPositiveThreshold] = useSetting(vadPositiveThresholdSetting); const [vadSensitivity, setVadSensitivity] = useSetting(vadPositiveThresholdSetting);
const [vadPositiveThresholdRaw, setVadPositiveThresholdRaw] = useState(vadPositiveThreshold); const [vadSensitivityRaw, setVadSensitivityRaw] = useState(vadSensitivity);
const [vadNegativeThreshold, setVadNegativeThreshold] = useSetting(vadNegativeThresholdSetting); const [vadAdvanced, setVadAdvanced] = useSetting(vadAdvancedEnabledSetting);
const [vadNegativeThresholdRaw, setVadNegativeThresholdRaw] = useState(vadNegativeThreshold); const vadState = !vadActive ? "disabled" : vadAdvanced ? "advanced" : "simple";
const setVadState = (s: "disabled" | "simple" | "advanced"): void => {
// Transient suppressor settings setVadActive(s !== "disabled");
const [transientEnabled, setTransientEnabled] = useSetting(transientSuppressorEnabledSetting); setVadAdvanced(s === "advanced");
const [transientThreshold, setTransientThreshold] = useSetting(transientThresholdSetting); };
const [transientThresholdRaw, setTransientThresholdRaw] = useState(transientThreshold); const [vadModeValue, setVadModeValue] = useSetting(vadModeSetting);
const [transientRelease, setTransientRelease] = useSetting(transientReleaseSetting); const [vadAdvOpen, setVadAdvOpen] = useSetting(vadAdvancedOpenThresholdSetting);
const [transientReleaseRaw, setTransientReleaseRaw] = useState(transientRelease); const [vadAdvOpenRaw, setVadAdvOpenRaw] = useState(vadAdvOpen);
const [vadAdvClose, setVadAdvClose] = useSetting(vadAdvancedCloseThresholdSetting);
const resetTransientDefaults = useCallback((): void => { const [vadAdvCloseRaw, setVadAdvCloseRaw] = useState(vadAdvClose);
const t = transientThresholdSetting.defaultValue; const [vadHold, setVadHold] = useSetting(vadHoldTimeSetting);
const r = transientReleaseSetting.defaultValue; const [vadHoldRaw, setVadHoldRaw] = useState(vadHold);
setTransientThreshold(t); setTransientThresholdRaw(t);
setTransientRelease(r); setTransientReleaseRaw(r);
}, [setTransientThreshold, setTransientRelease]);
const resetGateDefaults = useCallback((): void => {
const a = noiseGateAttackSetting.defaultValue;
const h = noiseGateHoldSetting.defaultValue;
const r = noiseGateReleaseSetting.defaultValue;
setNoiseGateAttack(a); setNoiseGateAttackRaw(a);
setNoiseGateHold(h); setNoiseGateHoldRaw(h);
setNoiseGateRelease(r); setNoiseGateReleaseRaw(r);
}, [setNoiseGateAttack, setNoiseGateHold, setNoiseGateRelease]);
const { available: isRageshakeAvailable } = useSubmitRageshake(); const { available: isRageshakeAvailable } = useSubmitRageshake();
@@ -220,234 +201,185 @@ export const SettingsModal: FC<Props> = ({
/> />
</div> </div>
</Form> </Form>
<div className={styles.noiseGateSection}> <div className={styles.vadSection}>
<Heading <Heading
type="body" type="body"
weight="semibold" weight="semibold"
size="sm" size="sm"
as="h4" as="h4"
className={styles.noiseGateHeading} className={styles.vadHeading}
> >
Noise Gate Voice Activity Detection
</Heading> </Heading>
<Separator className={styles.noiseGateSeparator} /> <Separator className={styles.vadSeparator} />
<FieldRow> <Form>
<InputField <InlineField
id="noiseGateEnabled" name={vadStateGroup}
type="checkbox" control={
label="Enable noise gate" <RadioControl
description="Suppress audio below a configurable threshold." checked={vadState === "disabled"}
checked={noiseGateEnabled} value="disabled"
onChange={(e: ChangeEvent<HTMLInputElement>): void => onChange={(): void => setVadState("disabled")}
setNoiseGateEnabled(e.target.checked) />
} }
/> >
</FieldRow> <Label>Disabled</Label>
{noiseGateEnabled && ( </InlineField>
<> <InlineField
<div className={`${styles.volumeSlider} ${styles.thresholdSlider}`}> name={vadStateGroup}
<span className={styles.sliderLabel}>Threshold</span> control={
<p>Gate opens above this level, closes below it.</p> <RadioControl
<NoiseLevelSlider checked={vadState === "simple"}
label="Noise gate threshold" value="simple"
value={noiseGateThresholdRaw} onChange={(): void => setVadState("simple")}
onValueChange={setNoiseGateThresholdRaw} />
onValueCommit={setNoiseGateThreshold} }
min={-100} >
max={0} <Label>Simple</Label>
step={1} </InlineField>
/> <InlineField
</div> name={vadStateGroup}
<div className={styles.advancedGate}> control={
<button <RadioControl
className={styles.advancedGateToggle} checked={vadState === "advanced"}
onClick={(): void => setShowAdvancedGate((v) => !v)} value="advanced"
> onChange={(): void => setVadState("advanced")}
{showAdvancedGate ? "▾" : "▸"} Advanced settings />
</button> }
{showAdvancedGate && ( >
<Label>Advanced</Label>
</InlineField>
</Form>
{vadState !== "disabled" && (
<>
{vadState === "simple" && (
<div className={styles.volumeSlider}>
<span className={styles.sliderLabel}>
Sensitivity: {Math.round(vadSensitivityRaw * 100)}%
</span>
<p>Higher values require more confident speech detection before opening.</p>
<Slider
label="VAD sensitivity"
value={vadSensitivityRaw}
onValueChange={setVadSensitivityRaw}
onValueCommit={setVadSensitivity}
min={0.1}
max={1.0}
step={0.05}
/>
</div>
)}
{vadState === "advanced" && (
<> <>
<div className={styles.volumeSlider}> <span className={styles.vadRampLabel}>Ramp profiles</span>
<label>Attack: {noiseGateAttackRaw} ms</label> <Form className={styles.vadRampForm}>
<p>How quickly the gate opens when signal exceeds threshold.</p> <InlineField
name={vadModeRadioGroup}
control={
<RadioControl
checked={vadModeValue === "loose"}
value="loose"
onChange={(): void => setVadModeValue("loose")}
/>
}
>
<Label>Loose</Label>
<HelpMessage>256 samples / 16 ms 12 ms open / 32 ms close ramp.</HelpMessage>
</InlineField>
<InlineField
name={vadModeRadioGroup}
control={
<RadioControl
checked={vadModeValue === "standard"}
value="standard"
onChange={(): void => setVadModeValue("standard")}
/>
}
>
<Label>Standard</Label>
<HelpMessage>256 samples / 16 ms 5 ms open / 20 ms close ramp.</HelpMessage>
</InlineField>
<InlineField
name={vadModeRadioGroup}
control={
<RadioControl
checked={vadModeValue === "aggressive"}
value="aggressive"
onChange={(): void => setVadModeValue("aggressive")}
/>
}
>
<Label>Aggressive</Label>
<HelpMessage>160 samples / 10 ms 1 ms open / 5 ms close ramp.</HelpMessage>
</InlineField>
</Form>
<div className={`${styles.volumeSlider} ${styles.vadSpacedSlider}`}>
<span className={styles.sliderLabel}>
Open threshold: {Math.round(vadAdvOpenRaw * 100)}%
</span>
<p>Minimum confidence required to open the gate.</p>
<Slider <Slider
label="Noise gate attack" label="VAD open threshold"
value={noiseGateAttackRaw} value={vadAdvOpenRaw}
onValueChange={setNoiseGateAttackRaw} onValueChange={setVadAdvOpenRaw}
onValueCommit={setNoiseGateAttack} onValueCommit={setVadAdvOpen}
min={1} min={0.1}
max={100} max={0.95}
step={1} step={0.05}
tooltip={false}
/> />
</div> </div>
<div className={styles.volumeSlider}> <div className={styles.volumeSlider}>
<label>Hold: {noiseGateHoldRaw} ms</label> <span className={styles.sliderLabel}>
<p>How long the gate stays open after signal drops below threshold.</p> Close threshold: {Math.round(vadAdvCloseRaw * 100)}%
</span>
<p>Probability must drop below this to start the hold/close sequence.</p>
<Slider <Slider
label="Noise gate hold" label="VAD close threshold"
value={noiseGateHoldRaw} value={vadAdvCloseRaw}
onValueChange={setNoiseGateHoldRaw} onValueChange={setVadAdvCloseRaw}
onValueCommit={setNoiseGateHold} onValueCommit={setVadAdvClose}
min={0.05}
max={0.9}
step={0.05}
/>
</div>
<div className={`${styles.volumeSlider} ${styles.vadSpacedSlider}`}>
<span className={styles.sliderLabel}>
Hold time: {vadHoldRaw} ms
</span>
<p>How long to keep the gate open after speech drops below the close threshold.</p>
<Slider
label="VAD hold time"
value={vadHoldRaw}
onValueChange={setVadHoldRaw}
onValueCommit={setVadHold}
min={0} min={0}
max={500} max={2000}
step={10} step={50}
tooltip={false}
/>
</div>
<div className={styles.volumeSlider}>
<label>Release: {noiseGateReleaseRaw} ms</label>
<p>How quickly the gate closes after hold expires.</p>
<Slider
label="Noise gate release"
value={noiseGateReleaseRaw}
onValueChange={setNoiseGateReleaseRaw}
onValueCommit={setNoiseGateRelease}
min={10}
max={500}
step={10}
tooltip={false}
/> />
</div> </div>
<div className={styles.restoreDefaults}> <div className={styles.restoreDefaults}>
<Button kind="secondary" size="sm" onClick={resetGateDefaults}> <Button
kind="secondary"
size="sm"
onClick={(): void => {
const defOpen = vadAdvancedOpenThresholdSetting.defaultValue;
const defClose = vadAdvancedCloseThresholdSetting.defaultValue;
const defHold = vadHoldTimeSetting.defaultValue;
setVadAdvOpen(defOpen); setVadAdvOpenRaw(defOpen);
setVadAdvClose(defClose); setVadAdvCloseRaw(defClose);
setVadHold(defHold); setVadHoldRaw(defHold);
setVadModeValue("standard");
}}
>
Restore defaults Restore defaults
</Button> </Button>
</div> </div>
</> </>
)} )}
</div>
</>
)}
</div>
<div className={styles.noiseGateSection}>
<Heading
type="body"
weight="semibold"
size="sm"
as="h4"
className={styles.noiseGateHeading}
>
Voice Activity Detection
</Heading>
<Separator className={styles.noiseGateSeparator} />
<FieldRow>
<InputField
id="vadEnabled"
type="checkbox"
label="Enable voice activity detection"
description="Uses TEN-VAD to mute audio when no speech is detected (~16 ms latency)."
checked={vadActive}
onChange={(e: ChangeEvent<HTMLInputElement>): void =>
setVadActive(e.target.checked)
}
/>
</FieldRow>
{vadActive && (
<>
<div className={`${styles.volumeSlider} ${styles.thresholdSlider}`}>
<span className={styles.sliderLabel}>Open threshold: {Math.round(vadPositiveThresholdRaw * 100)}%</span>
<p>How confident the model must be before opening the gate.</p>
<Slider
label="VAD open threshold"
value={vadPositiveThresholdRaw}
onValueChange={setVadPositiveThresholdRaw}
onValueCommit={setVadPositiveThreshold}
min={0.1}
max={0.9}
step={0.05}
tooltip={false}
/>
</div>
<div className={styles.volumeSlider}>
<span className={styles.sliderLabel}>Close threshold: {Math.round(vadNegativeThresholdRaw * 100)}%</span>
<p>How low the probability must drop before closing the gate.</p>
<Slider
label="VAD close threshold"
value={vadNegativeThresholdRaw}
onValueChange={setVadNegativeThresholdRaw}
onValueCommit={setVadNegativeThreshold}
min={0.05}
max={0.7}
step={0.05}
tooltip={false}
/>
</div>
<div className={styles.restoreDefaults}>
<Button
kind="secondary"
size="sm"
onClick={(): void => {
const pos = vadPositiveThresholdSetting.defaultValue;
const neg = vadNegativeThresholdSetting.defaultValue;
setVadPositiveThreshold(pos); setVadPositiveThresholdRaw(pos);
setVadNegativeThreshold(neg); setVadNegativeThresholdRaw(neg);
}}
>
Restore defaults
</Button>
</div>
</>
)}
</div>
<div className={styles.noiseGateSection}>
<Heading
type="body"
weight="semibold"
size="sm"
as="h4"
className={styles.noiseGateHeading}
>
Transient Suppressor
</Heading>
<Separator className={styles.noiseGateSeparator} />
<FieldRow>
<InputField
id="transientEnabled"
type="checkbox"
label="Enable transient suppressor"
description="Cut sudden loud impacts like desk hits or mic bumps."
checked={transientEnabled}
onChange={(e: ChangeEvent<HTMLInputElement>): void =>
setTransientEnabled(e.target.checked)
}
/>
</FieldRow>
{transientEnabled && (
<>
<div className={`${styles.volumeSlider} ${styles.thresholdSlider}`}>
<span className={styles.sliderLabel}>Sensitivity: {transientThresholdRaw} dB above background</span>
<p>Lower values catch more impacts; higher values only catch the loudest ones.</p>
<Slider
label="Transient threshold"
value={transientThresholdRaw}
onValueChange={setTransientThresholdRaw}
onValueCommit={setTransientThreshold}
min={8}
max={30}
step={1}
tooltip={false}
/>
</div>
<div className={styles.volumeSlider}>
<span className={styles.sliderLabel}>Release: {transientReleaseRaw} ms</span>
<p>How quickly audio returns after suppression.</p>
<Slider
label="Transient release"
value={transientReleaseRaw}
onValueChange={setTransientReleaseRaw}
onValueCommit={setTransientRelease}
min={20}
max={200}
step={10}
tooltip={false}
/>
</div>
<div className={styles.restoreDefaults}>
<Button kind="secondary" size="sm" onClick={resetTransientDefaults}>
Restore defaults
</Button>
</div>
</> </>
)} )}
</div> </div>

View File

@@ -129,42 +129,16 @@ export const alwaysShowIphoneEarpiece = new Setting<boolean>(
false, false,
); );
export const noiseGateEnabled = new Setting<boolean>(
"noise-gate-enabled",
false,
);
// Threshold in dBFS — gate opens above this, closes below it
export const noiseGateThreshold = new Setting<number>(
"noise-gate-threshold",
-60,
);
// Time in ms for the gate to fully open after signal exceeds threshold
export const noiseGateAttack = new Setting<number>("noise-gate-attack", 25);
// Time in ms the gate stays open after signal drops below threshold
export const noiseGateHold = new Setting<number>("noise-gate-hold", 200);
// Time in ms for the gate to fully close after hold expires
export const noiseGateRelease = new Setting<number>("noise-gate-release", 150);
export const vadEnabled = new Setting<boolean>("vad-enabled", false); export const vadEnabled = new Setting<boolean>("vad-enabled", false);
// Probability above which the VAD opens the gate (01) // Simple mode: single sensitivity slider (open threshold); close = open - 0.1
export const vadPositiveThreshold = new Setting<number>("vad-positive-threshold", 0.2); export const vadPositiveThreshold = new Setting<number>("vad-positive-threshold", 0.7);
// Probability below which the VAD closes the gate (01) // standard: 5ms/20ms aggressive: 1ms/5ms loose: 12ms/32ms
export const vadNegativeThreshold = new Setting<number>("vad-negative-threshold", 0.1); export const vadMode = new Setting<"standard" | "aggressive" | "loose">("vad-mode", "standard");
// Advanced settings (override simple mode when enabled)
export const transientSuppressorEnabled = new Setting<boolean>( export const vadAdvancedEnabled = new Setting<boolean>("vad-advanced-enabled", false);
"transient-suppressor-enabled", export const vadAdvancedOpenThreshold = new Setting<number>("vad-advanced-open-threshold", 0.7);
false, export const vadAdvancedCloseThreshold = new Setting<number>("vad-advanced-close-threshold", 0.6);
); export const vadHoldTime = new Setting<number>("vad-hold-time", 300);
// How many dB above the background RMS a peak must be to trigger suppression
export const transientThreshold = new Setting<number>(
"transient-suppressor-threshold",
15,
);
// Time in ms for suppression to fade after transient ends
export const transientRelease = new Setting<number>(
"transient-suppressor-release",
80,
);
export enum MatrixRTCMode { export enum MatrixRTCMode {
Legacy = "legacy", Legacy = "legacy",

View File

@@ -33,22 +33,18 @@ import {
} from "../../../livekit/TrackProcessorContext.tsx"; } from "../../../livekit/TrackProcessorContext.tsx";
import { getUrlParams } from "../../../UrlParams.ts"; import { getUrlParams } from "../../../UrlParams.ts";
import { import {
noiseGateEnabled,
noiseGateThreshold,
noiseGateAttack,
noiseGateHold,
noiseGateRelease,
transientSuppressorEnabled,
transientThreshold,
transientRelease,
vadEnabled, vadEnabled,
vadPositiveThreshold, vadPositiveThreshold,
vadNegativeThreshold, vadMode,
vadAdvancedEnabled,
vadAdvancedOpenThreshold,
vadAdvancedCloseThreshold,
vadHoldTime,
} from "../../../settings/settings.ts"; } from "../../../settings/settings.ts";
import { import {
type NoiseGateParams, type TenVadParams,
NoiseGateTransformer, TenVadTransformer,
} from "../../../livekit/NoiseGateTransformer.ts"; } from "../../../livekit/TenVadTransformer.ts";
import { observeTrackReference$ } from "../../observeTrackReference"; import { observeTrackReference$ } from "../../observeTrackReference";
import { type Connection } from "../remoteMembers/Connection.ts"; import { type Connection } from "../remoteMembers/Connection.ts";
import { ObservableScope } from "../../ObservableScope.ts"; import { ObservableScope } from "../../ObservableScope.ts";
@@ -93,7 +89,7 @@ export class Publisher {
// Setup track processor syncing (blur) // Setup track processor syncing (blur)
this.observeTrackProcessors(this.scope, room, trackerProcessorState$); this.observeTrackProcessors(this.scope, room, trackerProcessorState$);
// Setup noise gate on the local microphone track // Setup noise gate on the local microphone track
this.applyNoiseGate(this.scope, room); this.applyTenVad(this.scope, room);
// Observe media device changes and update LiveKit active devices accordingly // Observe media device changes and update LiveKit active devices accordingly
this.observeMediaDevices(this.scope, devices, controlledAudioDevices); this.observeMediaDevices(this.scope, devices, controlledAudioDevices);
@@ -421,7 +417,7 @@ export class Publisher {
}); });
} }
private applyNoiseGate(scope: ObservableScope, room: LivekitRoom): void { private applyTenVad(scope: ObservableScope, room: LivekitRoom): void {
// Observe the local microphone track // Observe the local microphone track
const audioTrack$ = scope.behavior( const audioTrack$ = scope.behavior(
observeTrackReference$( observeTrackReference$(
@@ -436,48 +432,55 @@ export class Publisher {
null, null,
); );
let transformer: NoiseGateTransformer | null = null; let transformer: TenVadTransformer | null = null;
let audioCtx: AudioContext | null = null; let audioCtx: AudioContext | null = null;
const currentParams = (): NoiseGateParams => ({ const currentParams = (): TenVadParams => {
noiseGateActive: noiseGateEnabled.getValue(), const isAdvanced = vadAdvancedEnabled.getValue();
threshold: noiseGateThreshold.getValue(), if (isAdvanced) {
attackMs: noiseGateAttack.getValue(), return {
holdMs: noiseGateHold.getValue(), vadEnabled: vadEnabled.getValue(),
releaseMs: noiseGateRelease.getValue(), vadPositiveThreshold: vadAdvancedOpenThreshold.getValue(),
transientEnabled: transientSuppressorEnabled.getValue(), vadNegativeThreshold: vadAdvancedCloseThreshold.getValue(),
transientThresholdDb: transientThreshold.getValue(), vadMode: vadMode.getValue(),
transientReleaseMs: transientRelease.getValue(), holdMs: vadHoldTime.getValue(),
vadEnabled: vadEnabled.getValue(), };
vadPositiveThreshold: vadPositiveThreshold.getValue(), }
vadNegativeThreshold: vadNegativeThreshold.getValue(), const openT = vadPositiveThreshold.getValue();
}); return {
vadEnabled: vadEnabled.getValue(),
vadPositiveThreshold: openT,
vadNegativeThreshold: Math.max(0, openT - 0.1),
vadMode: "standard",
holdMs: 0,
};
};
// Attach / detach processor when any processing feature changes or the track changes. // Attach / detach processor when VAD is toggled or the track changes.
combineLatest([audioTrack$, noiseGateEnabled.value$, vadEnabled.value$, transientSuppressorEnabled.value$]) combineLatest([audioTrack$, vadEnabled.value$])
.pipe(scope.bind()) .pipe(scope.bind())
.subscribe(([audioTrack, ngEnabled, vadActive, transientActive]) => { .subscribe(([audioTrack, vadActive]) => {
if (!audioTrack) return; if (!audioTrack) return;
const shouldAttach = ngEnabled || vadActive || transientActive; const shouldAttach = vadActive;
if (shouldAttach && !audioTrack.getProcessor()) { if (shouldAttach && !audioTrack.getProcessor()) {
const params = currentParams(); const params = currentParams();
this.logger.info("[NoiseGate] attaching processor, params:", params); this.logger.info("[TenVad] attaching processor, params:", params);
transformer = new NoiseGateTransformer(params); transformer = new TenVadTransformer(params);
audioCtx = new AudioContext(); audioCtx = new AudioContext();
this.logger.info("[NoiseGate] AudioContext state before resume:", audioCtx.state); this.logger.info("[TenVad] AudioContext state before resume:", audioCtx.state);
// eslint-disable-next-line @typescript-eslint/no-explicit-any // eslint-disable-next-line @typescript-eslint/no-explicit-any
(audioTrack as any).setAudioContext(audioCtx); (audioTrack as any).setAudioContext(audioCtx);
audioCtx.resume().then(async () => { audioCtx.resume().then(async () => {
this.logger.info("[NoiseGate] AudioContext state after resume:", audioCtx?.state); this.logger.info("[TenVad] AudioContext state after resume:", audioCtx?.state);
// eslint-disable-next-line @typescript-eslint/no-explicit-any // eslint-disable-next-line @typescript-eslint/no-explicit-any
return audioTrack.setProcessor(transformer as any); return audioTrack.setProcessor(transformer as any);
}).then(() => { }).then(() => {
this.logger.info("[NoiseGate] setProcessor resolved"); this.logger.info("[TenVad] setProcessor resolved");
}).catch((e: unknown) => { }).catch((e: unknown) => {
this.logger.error("[NoiseGate] setProcessor failed", e); this.logger.error("[TenVad] setProcessor failed", e);
}); });
} else if (!shouldAttach && audioTrack.getProcessor()) { } else if (!shouldAttach && audioTrack.getProcessor()) {
this.logger.info("[NoiseGate] removing processor"); this.logger.info("[TenVad] removing processor");
void audioTrack.stopProcessor(); void audioTrack.stopProcessor();
void audioCtx?.close(); void audioCtx?.close();
audioCtx = null; audioCtx = null;
@@ -485,44 +488,29 @@ export class Publisher {
// eslint-disable-next-line @typescript-eslint/no-explicit-any // eslint-disable-next-line @typescript-eslint/no-explicit-any
(audioTrack as any).setAudioContext(undefined); (audioTrack as any).setAudioContext(undefined);
} else if (shouldAttach && audioTrack.getProcessor()) { } else if (shouldAttach && audioTrack.getProcessor()) {
// Processor already attached — push updated params (e.g. noiseGateActive toggled) // Processor already attached — push updated params (e.g. vadActive toggled)
transformer?.updateParams(currentParams()); transformer?.updateParams(currentParams());
} else { } else {
this.logger.info( this.logger.info(
"[NoiseGate] tick — ngEnabled:", ngEnabled, "[TenVad] tick — vadActive:", vadActive,
"vadActive:", vadActive,
"hasProcessor:", !!audioTrack.getProcessor(), "hasProcessor:", !!audioTrack.getProcessor(),
); );
} }
}); });
// Push all param changes (noise gate + VAD) to the live worklet. // Push VAD param changes to the live worklet.
combineLatest([ combineLatest([
noiseGateEnabled.value$,
noiseGateThreshold.value$,
noiseGateAttack.value$,
noiseGateHold.value$,
noiseGateRelease.value$,
transientSuppressorEnabled.value$,
transientThreshold.value$,
transientRelease.value$,
vadEnabled.value$, vadEnabled.value$,
vadPositiveThreshold.value$, vadPositiveThreshold.value$,
vadNegativeThreshold.value$, vadMode.value$,
vadAdvancedEnabled.value$,
vadAdvancedOpenThreshold.value$,
vadAdvancedCloseThreshold.value$,
vadHoldTime.value$,
]) ])
.pipe(scope.bind()) .pipe(scope.bind())
.subscribe(([ .subscribe(() => {
noiseGateActive, threshold, attackMs, holdMs, releaseMs, transformer?.updateParams(currentParams());
transientEnabled, transientThresholdDb, transientReleaseMs,
vadActive, vadPos, vadNeg,
]) => {
transformer?.updateParams({
noiseGateActive, threshold, attackMs, holdMs, releaseMs,
transientEnabled, transientThresholdDb, transientReleaseMs,
vadEnabled: vadActive,
vadPositiveThreshold: vadPos,
vadNegativeThreshold: vadNeg,
});
}); });
} }
@@ -542,5 +530,4 @@ export class Publisher {
); );
trackProcessorSync(scope, track$, trackerProcessorState$); trackProcessorSync(scope, track$, trackerProcessorState$);
} }
} }

View File

@@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details.
import { import {
loadEnv, loadEnv,
PluginOption,
searchForWorkspaceRoot, searchForWorkspaceRoot,
type ConfigEnv, type ConfigEnv,
type UserConfig, type UserConfig,
@@ -33,8 +34,7 @@ export default ({
// In future we might be able to do what is needed via code splitting at // In future we might be able to do what is needed via code splitting at
// build time. // build time.
process.env.VITE_PACKAGE = packageType ?? "full"; process.env.VITE_PACKAGE = packageType ?? "full";
const plugins: PluginOption[] = [
const plugins = [
react(), react(),
svgrPlugin({ svgrPlugin({
svgrOptions: { svgrOptions: {