WIP: renaming old noise gate file names to ten vad related file names
Some checks failed
Build / build_full_element_call (pull_request) Has been cancelled
Build / build_embedded_element_call (pull_request) Has been cancelled
Build / build_sdk_element_call (pull_request) Has been cancelled
Lint, format & type check / Lint, format & type check (pull_request) Has been cancelled
Build & publish embedded packages for releases / Versioning (pull_request) Has been cancelled
Test / Run unit tests (pull_request) Has been cancelled
Test / Run end-to-end tests (pull_request) Has been cancelled
GitHub Actions Security Analysis with zizmor 🌈 / Run zizmor 🌈 (pull_request) Has been cancelled
Prevent blocked / Prevent blocked (pull_request_target) Has been cancelled
Build / deploy_develop (pull_request) Has been cancelled
Build / docker_for_develop (pull_request) Has been cancelled
Build & publish embedded packages for releases / build_element_call (pull_request) Has been cancelled
Build & publish embedded packages for releases / Publish tarball (pull_request) Has been cancelled
Build & publish embedded packages for releases / Publish NPM (pull_request) Has been cancelled
Build & publish embedded packages for releases / Publish Android AAR (pull_request) Has been cancelled
Build & publish embedded packages for releases / Publish SwiftPM Library (pull_request) Has been cancelled
Build & publish embedded packages for releases / Update release notes (pull_request) Has been cancelled
Some checks failed
Build / build_full_element_call (pull_request) Has been cancelled
Build / build_embedded_element_call (pull_request) Has been cancelled
Build / build_sdk_element_call (pull_request) Has been cancelled
Lint, format & type check / Lint, format & type check (pull_request) Has been cancelled
Build & publish embedded packages for releases / Versioning (pull_request) Has been cancelled
Test / Run unit tests (pull_request) Has been cancelled
Test / Run end-to-end tests (pull_request) Has been cancelled
GitHub Actions Security Analysis with zizmor 🌈 / Run zizmor 🌈 (pull_request) Has been cancelled
Prevent blocked / Prevent blocked (pull_request_target) Has been cancelled
Build / deploy_develop (pull_request) Has been cancelled
Build / docker_for_develop (pull_request) Has been cancelled
Build & publish embedded packages for releases / build_element_call (pull_request) Has been cancelled
Build & publish embedded packages for releases / Publish tarball (pull_request) Has been cancelled
Build & publish embedded packages for releases / Publish NPM (pull_request) Has been cancelled
Build & publish embedded packages for releases / Publish Android AAR (pull_request) Has been cancelled
Build & publish embedded packages for releases / Publish SwiftPM Library (pull_request) Has been cancelled
Build & publish embedded packages for releases / Update release notes (pull_request) Has been cancelled
This commit is contained in:
@@ -23,7 +23,7 @@ declare function registerProcessor(
|
|||||||
processorCtor: new (...args: any[]) => AudioWorkletProcessor,
|
processorCtor: new (...args: any[]) => AudioWorkletProcessor,
|
||||||
): void;
|
): void;
|
||||||
|
|
||||||
interface NoiseGateParams {
|
interface TenVadParams {
|
||||||
// TEN-VAD params
|
// TEN-VAD params
|
||||||
vadEnabled: boolean;
|
vadEnabled: boolean;
|
||||||
vadPositiveThreshold: number; // open gate when prob >= this (0–1)
|
vadPositiveThreshold: number; // open gate when prob >= this (0–1)
|
||||||
@@ -169,7 +169,7 @@ class TenVADRuntime {
|
|||||||
* Asymmetric ramp: 5 ms open (minimise speech onset masking), 20 ms close
|
* Asymmetric ramp: 5 ms open (minimise speech onset masking), 20 ms close
|
||||||
* (de-click on silence).
|
* (de-click on silence).
|
||||||
*/
|
*/
|
||||||
class NoiseGateProcessor extends AudioWorkletProcessor {
|
class TenVadProcessor extends AudioWorkletProcessor {
|
||||||
// VAD gate state
|
// VAD gate state
|
||||||
private vadGateOpen = true; // starts open; TEN-VAD closes it on first silent frame
|
private vadGateOpen = true; // starts open; TEN-VAD closes it on first silent frame
|
||||||
private vadAttenuation = 1.0;
|
private vadAttenuation = 1.0;
|
||||||
@@ -212,23 +212,23 @@ class NoiseGateProcessor extends AudioWorkletProcessor {
|
|||||||
this.tenVadRuntime = new TenVADRuntime(this.tenVadModule, 256, 0.5);
|
this.tenVadRuntime = new TenVADRuntime(this.tenVadModule, 256, 0.5);
|
||||||
this.port.postMessage({
|
this.port.postMessage({
|
||||||
type: "log",
|
type: "log",
|
||||||
msg: "[NoiseGate worklet] TEN-VAD runtime initialized, decRatio=" + this.decRatio,
|
msg: "[TenVad worklet] TEN-VAD runtime initialized, decRatio=" + this.decRatio,
|
||||||
});
|
});
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
this.port.postMessage({
|
this.port.postMessage({
|
||||||
type: "log",
|
type: "log",
|
||||||
msg: "[NoiseGate worklet] TEN-VAD init failed: " + String(e),
|
msg: "[TenVad worklet] TEN-VAD init failed: " + String(e),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
this.port.onmessage = (
|
this.port.onmessage = (
|
||||||
e: MessageEvent<NoiseGateParams | VADGateMessage>,
|
e: MessageEvent<TenVadParams | VADGateMessage>,
|
||||||
): void => {
|
): void => {
|
||||||
if ((e.data as VADGateMessage).type === "vad-gate") {
|
if ((e.data as VADGateMessage).type === "vad-gate") {
|
||||||
this.vadGateOpen = (e.data as VADGateMessage).open;
|
this.vadGateOpen = (e.data as VADGateMessage).open;
|
||||||
} else {
|
} else {
|
||||||
this.updateParams(e.data as NoiseGateParams);
|
this.updateParams(e.data as TenVadParams);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
@@ -242,11 +242,11 @@ class NoiseGateProcessor extends AudioWorkletProcessor {
|
|||||||
|
|
||||||
this.port.postMessage({
|
this.port.postMessage({
|
||||||
type: "log",
|
type: "log",
|
||||||
msg: "[NoiseGate worklet] constructor called, sampleRate=" + sampleRate,
|
msg: "[TenVad worklet] constructor called, sampleRate=" + sampleRate,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private updateParams(p: NoiseGateParams): void {
|
private updateParams(p: TenVadParams): void {
|
||||||
this.vadEnabled = p.vadEnabled ?? false;
|
this.vadEnabled = p.vadEnabled ?? false;
|
||||||
this.vadPositiveThreshold = p.vadPositiveThreshold ?? 0.5;
|
this.vadPositiveThreshold = p.vadPositiveThreshold ?? 0.5;
|
||||||
this.vadNegativeThreshold = p.vadNegativeThreshold ?? 0.3;
|
this.vadNegativeThreshold = p.vadNegativeThreshold ?? 0.3;
|
||||||
@@ -273,7 +273,7 @@ class NoiseGateProcessor extends AudioWorkletProcessor {
|
|||||||
try {
|
try {
|
||||||
this.tenVadRuntime = new TenVADRuntime(this.tenVadModule, newHopSize, 0.5);
|
this.tenVadRuntime = new TenVADRuntime(this.tenVadModule, newHopSize, 0.5);
|
||||||
} catch (e) {
|
} catch (e) {
|
||||||
this.port.postMessage({ type: "log", msg: "[NoiseGate worklet] TEN-VAD recreate failed: " + String(e) });
|
this.port.postMessage({ type: "log", msg: "[TenVad worklet] TEN-VAD recreate failed: " + String(e) });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
this.vadHopSize = newHopSize;
|
this.vadHopSize = newHopSize;
|
||||||
@@ -287,7 +287,7 @@ class NoiseGateProcessor extends AudioWorkletProcessor {
|
|||||||
if (!this.vadEnabled) this.vadGateOpen = true;
|
if (!this.vadEnabled) this.vadGateOpen = true;
|
||||||
this.port.postMessage({
|
this.port.postMessage({
|
||||||
type: "log",
|
type: "log",
|
||||||
msg: "[NoiseGate worklet] params updated: vadEnabled=" + p.vadEnabled
|
msg: "[TenVad worklet] params updated: vadEnabled=" + p.vadEnabled
|
||||||
+ " vadPos=" + p.vadPositiveThreshold
|
+ " vadPos=" + p.vadPositiveThreshold
|
||||||
+ " vadNeg=" + p.vadNegativeThreshold
|
+ " vadNeg=" + p.vadNegativeThreshold
|
||||||
+ " vadMode=" + newMode
|
+ " vadMode=" + newMode
|
||||||
@@ -379,7 +379,7 @@ class NoiseGateProcessor extends AudioWorkletProcessor {
|
|||||||
if (this.logCounter % 375 === 0) {
|
if (this.logCounter % 375 === 0) {
|
||||||
this.port.postMessage({
|
this.port.postMessage({
|
||||||
type: "log",
|
type: "log",
|
||||||
msg: "[NoiseGate worklet] vadOpen=" + this.vadGateOpen
|
msg: "[TenVad worklet] vadOpen=" + this.vadGateOpen
|
||||||
+ " vadAtten=" + this.vadAttenuation.toFixed(3),
|
+ " vadAtten=" + this.vadAttenuation.toFixed(3),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
@@ -388,4 +388,4 @@ class NoiseGateProcessor extends AudioWorkletProcessor {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
registerProcessor("noise-gate-processor", NoiseGateProcessor);
|
registerProcessor("ten-vad-processor", TenVadProcessor);
|
||||||
@@ -9,11 +9,11 @@ import { type Track } from "livekit-client";
|
|||||||
import { logger } from "matrix-js-sdk/lib/logger";
|
import { logger } from "matrix-js-sdk/lib/logger";
|
||||||
// ?worker&url tells Vite to compile the TypeScript worklet and return its URL.
|
// ?worker&url tells Vite to compile the TypeScript worklet and return its URL.
|
||||||
// Without this, Vite copies the .ts file verbatim and the browser rejects it.
|
// Without this, Vite copies the .ts file verbatim and the browser rejects it.
|
||||||
import compiledWorkletUrl from "./NoiseGateProcessor.worklet.ts?worker&url";
|
import compiledWorkletUrl from "./TenVadProcessor.worklet.ts?worker&url";
|
||||||
|
|
||||||
const log = logger.getChild("[NoiseGateTransformer]");
|
const log = logger.getChild("[TenVadTransformer]");
|
||||||
|
|
||||||
export interface NoiseGateParams {
|
export interface TenVadParams {
|
||||||
// TEN-VAD params — processed entirely inside the AudioWorklet
|
// TEN-VAD params — processed entirely inside the AudioWorklet
|
||||||
vadEnabled: boolean;
|
vadEnabled: boolean;
|
||||||
vadPositiveThreshold: number; // open gate when prob >= this (0–1)
|
vadPositiveThreshold: number; // open gate when prob >= this (0–1)
|
||||||
@@ -74,16 +74,16 @@ function getTenVADModule(): Promise<WebAssembly.Module> {
|
|||||||
* Audio graph: sourceNode → workletNode → destinationNode
|
* Audio graph: sourceNode → workletNode → destinationNode
|
||||||
* processedTrack is destinationNode.stream.getAudioTracks()[0]
|
* processedTrack is destinationNode.stream.getAudioTracks()[0]
|
||||||
*/
|
*/
|
||||||
export class NoiseGateTransformer implements AudioTrackProcessor {
|
export class TenVadTransformer implements AudioTrackProcessor {
|
||||||
public readonly name = "noise-gate";
|
public readonly name = "noise-gate";
|
||||||
public processedTrack?: MediaStreamTrack;
|
public processedTrack?: MediaStreamTrack;
|
||||||
|
|
||||||
private workletNode?: AudioWorkletNode;
|
private workletNode?: AudioWorkletNode;
|
||||||
private sourceNode?: MediaStreamAudioSourceNode;
|
private sourceNode?: MediaStreamAudioSourceNode;
|
||||||
private destinationNode?: MediaStreamAudioDestinationNode;
|
private destinationNode?: MediaStreamAudioDestinationNode;
|
||||||
private params: NoiseGateParams;
|
private params: TenVadParams;
|
||||||
|
|
||||||
public constructor(params: NoiseGateParams) {
|
public constructor(params: TenVadParams) {
|
||||||
this.params = { ...params };
|
this.params = { ...params };
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -107,7 +107,7 @@ export class NoiseGateTransformer implements AudioTrackProcessor {
|
|||||||
|
|
||||||
this.workletNode = new AudioWorkletNode(
|
this.workletNode = new AudioWorkletNode(
|
||||||
audioContext,
|
audioContext,
|
||||||
"noise-gate-processor",
|
"ten-vad-processor",
|
||||||
{
|
{
|
||||||
processorOptions: {
|
processorOptions: {
|
||||||
tenVadModule,
|
tenVadModule,
|
||||||
@@ -150,7 +150,7 @@ export class NoiseGateTransformer implements AudioTrackProcessor {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/** Push updated gate/VAD parameters to the running worklet. */
|
/** Push updated gate/VAD parameters to the running worklet. */
|
||||||
public updateParams(params: NoiseGateParams): void {
|
public updateParams(params: TenVadParams): void {
|
||||||
this.params = { ...params };
|
this.params = { ...params };
|
||||||
this.sendParams();
|
this.sendParams();
|
||||||
}
|
}
|
||||||
@@ -35,16 +35,16 @@ Please see LICENSE in the repository root for full details.
|
|||||||
color: var(--cpd-color-text-secondary);
|
color: var(--cpd-color-text-secondary);
|
||||||
}
|
}
|
||||||
|
|
||||||
.noiseGateSection {
|
.vadSection {
|
||||||
margin-block-start: var(--cpd-space-6x);
|
margin-block-start: var(--cpd-space-6x);
|
||||||
}
|
}
|
||||||
|
|
||||||
.noiseGateHeading {
|
.vadHeading {
|
||||||
color: var(--cpd-color-text-secondary);
|
color: var(--cpd-color-text-secondary);
|
||||||
margin-block: var(--cpd-space-3x) 0;
|
margin-block: var(--cpd-space-3x) 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
.noiseGateSeparator {
|
.vadSeparator {
|
||||||
margin-block: 6px var(--cpd-space-4x);
|
margin-block: 6px var(--cpd-space-4x);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -201,17 +201,17 @@ export const SettingsModal: FC<Props> = ({
|
|||||||
/>
|
/>
|
||||||
</div>
|
</div>
|
||||||
</Form>
|
</Form>
|
||||||
<div className={styles.noiseGateSection}>
|
<div className={styles.vadSection}>
|
||||||
<Heading
|
<Heading
|
||||||
type="body"
|
type="body"
|
||||||
weight="semibold"
|
weight="semibold"
|
||||||
size="sm"
|
size="sm"
|
||||||
as="h4"
|
as="h4"
|
||||||
className={styles.noiseGateHeading}
|
className={styles.vadHeading}
|
||||||
>
|
>
|
||||||
Voice Activity Detection
|
Voice Activity Detection
|
||||||
</Heading>
|
</Heading>
|
||||||
<Separator className={styles.noiseGateSeparator} />
|
<Separator className={styles.vadSeparator} />
|
||||||
<Form>
|
<Form>
|
||||||
<InlineField
|
<InlineField
|
||||||
name={vadStateGroup}
|
name={vadStateGroup}
|
||||||
|
|||||||
@@ -42,9 +42,9 @@ import {
|
|||||||
vadHoldTime,
|
vadHoldTime,
|
||||||
} from "../../../settings/settings.ts";
|
} from "../../../settings/settings.ts";
|
||||||
import {
|
import {
|
||||||
type NoiseGateParams,
|
type TenVadParams,
|
||||||
NoiseGateTransformer,
|
TenVadTransformer,
|
||||||
} from "../../../livekit/NoiseGateTransformer.ts";
|
} from "../../../livekit/TenVadTransformer.ts";
|
||||||
import { observeTrackReference$ } from "../../observeTrackReference";
|
import { observeTrackReference$ } from "../../observeTrackReference";
|
||||||
import { type Connection } from "../remoteMembers/Connection.ts";
|
import { type Connection } from "../remoteMembers/Connection.ts";
|
||||||
import { ObservableScope } from "../../ObservableScope.ts";
|
import { ObservableScope } from "../../ObservableScope.ts";
|
||||||
@@ -89,7 +89,7 @@ export class Publisher {
|
|||||||
// Setup track processor syncing (blur)
|
// Setup track processor syncing (blur)
|
||||||
this.observeTrackProcessors(this.scope, room, trackerProcessorState$);
|
this.observeTrackProcessors(this.scope, room, trackerProcessorState$);
|
||||||
// Setup noise gate on the local microphone track
|
// Setup noise gate on the local microphone track
|
||||||
this.applyNoiseGate(this.scope, room);
|
this.applyTenVad(this.scope, room);
|
||||||
// Observe media device changes and update LiveKit active devices accordingly
|
// Observe media device changes and update LiveKit active devices accordingly
|
||||||
this.observeMediaDevices(this.scope, devices, controlledAudioDevices);
|
this.observeMediaDevices(this.scope, devices, controlledAudioDevices);
|
||||||
|
|
||||||
@@ -417,7 +417,7 @@ export class Publisher {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
private applyNoiseGate(scope: ObservableScope, room: LivekitRoom): void {
|
private applyTenVad(scope: ObservableScope, room: LivekitRoom): void {
|
||||||
// Observe the local microphone track
|
// Observe the local microphone track
|
||||||
const audioTrack$ = scope.behavior(
|
const audioTrack$ = scope.behavior(
|
||||||
observeTrackReference$(
|
observeTrackReference$(
|
||||||
@@ -432,10 +432,10 @@ export class Publisher {
|
|||||||
null,
|
null,
|
||||||
);
|
);
|
||||||
|
|
||||||
let transformer: NoiseGateTransformer | null = null;
|
let transformer: TenVadTransformer | null = null;
|
||||||
let audioCtx: AudioContext | null = null;
|
let audioCtx: AudioContext | null = null;
|
||||||
|
|
||||||
const currentParams = (): NoiseGateParams => {
|
const currentParams = (): TenVadParams => {
|
||||||
const isAdvanced = vadAdvancedEnabled.getValue();
|
const isAdvanced = vadAdvancedEnabled.getValue();
|
||||||
if (isAdvanced) {
|
if (isAdvanced) {
|
||||||
return {
|
return {
|
||||||
@@ -464,23 +464,23 @@ export class Publisher {
|
|||||||
const shouldAttach = vadActive;
|
const shouldAttach = vadActive;
|
||||||
if (shouldAttach && !audioTrack.getProcessor()) {
|
if (shouldAttach && !audioTrack.getProcessor()) {
|
||||||
const params = currentParams();
|
const params = currentParams();
|
||||||
this.logger.info("[NoiseGate] attaching processor, params:", params);
|
this.logger.info("[TenVad] attaching processor, params:", params);
|
||||||
transformer = new NoiseGateTransformer(params);
|
transformer = new TenVadTransformer(params);
|
||||||
audioCtx = new AudioContext();
|
audioCtx = new AudioContext();
|
||||||
this.logger.info("[NoiseGate] AudioContext state before resume:", audioCtx.state);
|
this.logger.info("[TenVad] AudioContext state before resume:", audioCtx.state);
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
(audioTrack as any).setAudioContext(audioCtx);
|
(audioTrack as any).setAudioContext(audioCtx);
|
||||||
audioCtx.resume().then(async () => {
|
audioCtx.resume().then(async () => {
|
||||||
this.logger.info("[NoiseGate] AudioContext state after resume:", audioCtx?.state);
|
this.logger.info("[TenVad] AudioContext state after resume:", audioCtx?.state);
|
||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
return audioTrack.setProcessor(transformer as any);
|
return audioTrack.setProcessor(transformer as any);
|
||||||
}).then(() => {
|
}).then(() => {
|
||||||
this.logger.info("[NoiseGate] setProcessor resolved");
|
this.logger.info("[TenVad] setProcessor resolved");
|
||||||
}).catch((e: unknown) => {
|
}).catch((e: unknown) => {
|
||||||
this.logger.error("[NoiseGate] setProcessor failed", e);
|
this.logger.error("[TenVad] setProcessor failed", e);
|
||||||
});
|
});
|
||||||
} else if (!shouldAttach && audioTrack.getProcessor()) {
|
} else if (!shouldAttach && audioTrack.getProcessor()) {
|
||||||
this.logger.info("[NoiseGate] removing processor");
|
this.logger.info("[TenVad] removing processor");
|
||||||
void audioTrack.stopProcessor();
|
void audioTrack.stopProcessor();
|
||||||
void audioCtx?.close();
|
void audioCtx?.close();
|
||||||
audioCtx = null;
|
audioCtx = null;
|
||||||
@@ -488,11 +488,11 @@ export class Publisher {
|
|||||||
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
// eslint-disable-next-line @typescript-eslint/no-explicit-any
|
||||||
(audioTrack as any).setAudioContext(undefined);
|
(audioTrack as any).setAudioContext(undefined);
|
||||||
} else if (shouldAttach && audioTrack.getProcessor()) {
|
} else if (shouldAttach && audioTrack.getProcessor()) {
|
||||||
// Processor already attached — push updated params (e.g. noiseGateActive toggled)
|
// Processor already attached — push updated params (e.g. vadActive toggled)
|
||||||
transformer?.updateParams(currentParams());
|
transformer?.updateParams(currentParams());
|
||||||
} else {
|
} else {
|
||||||
this.logger.info(
|
this.logger.info(
|
||||||
"[NoiseGate] tick — vadActive:", vadActive,
|
"[TenVad] tick — vadActive:", vadActive,
|
||||||
"hasProcessor:", !!audioTrack.getProcessor(),
|
"hasProcessor:", !!audioTrack.getProcessor(),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user