Skip to content

Commit fb1c35a

Browse files
csukuangfjcsukuangfj
authored
Add JavaScript (WebAssembly) API for ten-vad (#2382)
Add support for the ten-vad model alongside silero-vad in the WebAssembly VAD API, update the UI and documentation, and extend examples and CI workflows to handle the new model. - Extend C++ bindings and printing logic to include ten-vad configuration. - Implement JavaScript init/free routines and runtime detection for ten-vad. - Update UI layout, README assets, example scripts, and CI workflow to support ten-vad.
1 parent f2df116 commit fb1c35a

File tree

10 files changed

+427
-25
lines changed

10 files changed

+427
-25
lines changed

‎.github/workflows/wasm-simd-hf-space-ten-vad.yaml

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
name: wasm-simd-hf-space-ten-vad
2+
3+
on:
4+
push:
5+
branches:
6+
- wasm
7+
- wasm-ten-vad
8+
tags:
9+
- 'v[0-9]+.[0-9]+.[0-9]+*'
10+
11+
workflow_dispatch:
12+
13+
concurrency:
14+
group: wasm-simd-hf-space-ten-vad-${{ github.ref }}
15+
cancel-in-progress: true
16+
17+
jobs:
18+
wasm-simd-hf-space-ten-vad:
19+
runs-on: ${{ matrix.os }}
20+
strategy:
21+
fail-fast: false
22+
matrix:
23+
os: [ubuntu-latest]
24+
25+
steps:
26+
- uses: actions/checkout@v4
27+
with:
28+
fetch-depth: 0
29+
30+
- name: Update version
31+
shell: bash
32+
run: |
33+
./new-release.sh
34+
git diff .
35+
36+
- name: Install emsdk
37+
uses: mymindstorm/setup-emsdk@v14
38+
with:
39+
version: 3.1.53
40+
actions-cache-folder: 'emsdk-cache'
41+
42+
- name: View emsdk version
43+
shell: bash
44+
run: |
45+
emcc -v
46+
echo "--------------------"
47+
emcc --check
48+
49+
- name: Download model files
50+
shell: bash
51+
run: |
52+
cd wasm/vad/assets
53+
ls -lh
54+
echo "----------"
55+
curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/ten-vad.onnx
56+
ls -lh
57+
cd ..
58+
sed -i.bak "s|.*(with <a .*| (with <a href="https://github.com/TEN-framework/ten-vad">ten-vad</a>)|" ./index.html
59+
git diff .
60+
61+
- name: Build sherpa-onnx for WebAssembly
62+
shell: bash
63+
run: |
64+
./build-wasm-simd-vad.sh
65+
66+
- name: collect files
67+
shell: bash
68+
run: |
69+
SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2)
70+
71+
dst=sherpa-onnx-wasm-simd-${SHERPA_ONNX_VERSION}-ten-vad
72+
mv build-wasm-simd-vad/install/bin/wasm/vad $dst
73+
ls -lh $dst
74+
tar cjfv $dst.tar.bz2 ./$dst
75+
76+
- name: Upload wasm files
77+
uses: actions/upload-artifact@v4
78+
with:
79+
name: sherpa-onnx-wasm-simd-ten-vad
80+
path: ./sherpa-onnx-wasm-simd-*.tar.bz2
81+
82+
- name: Release
83+
if: (github.repository_owner == 'csukuangfj' || github.repository_owner == 'k2-fsa') && github.event_name == 'push' && contains(github.ref, 'refs/tags/')
84+
uses: svenstaro/upload-release-action@v2
85+
with:
86+
file_glob: true
87+
overwrite: true
88+
file: ./*.tar.bz2
89+
90+
- name: Publish to ModelScope
91+
# if: false
92+
env:
93+
MS_TOKEN: ${{ secrets.MODEL_SCOPE_GIT_TOKEN }}
94+
uses: nick-fields/retry@v2
95+
with:
96+
max_attempts: 20
97+
timeout_seconds: 200
98+
shell: bash
99+
command: |
100+
SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2)
101+
102+
git config --global user.email "csukuangfj@gmail.com"
103+
git config --global user.name "Fangjun Kuang"
104+
105+
rm -rf ms
106+
export GIT_LFS_SKIP_SMUDGE=1
107+
export GIT_CLONE_PROTECTION_ACTIVE=false
108+
109+
git clone https://www.modelscope.cn/studios/csukuangfj/web-assembly-ten-vad-sherpa-onnx.git ms
110+
cd ms
111+
rm -fv *.js
112+
rm -fv *.data
113+
git fetch
114+
git pull
115+
git merge -m "merge remote" --ff origin main
116+
117+
cp -v ../sherpa-onnx-wasm-simd-${SHERPA_ONNX_VERSION}-ten-vad/* .
118+
119+
git status
120+
git lfs track "*.data"
121+
git lfs track "*.wasm"
122+
ls -lh
123+
124+
git add .
125+
git commit -m "update model"
126+
git push https://oauth2:${MS_TOKEN}@www.modelscope.cn/studios/csukuangfj/web-assembly-ten-vad-sherpa-onnx.git
127+
128+
- name: Publish to huggingface
129+
env:
130+
HF_TOKEN: ${{ secrets.HF_TOKEN }}
131+
uses: nick-fields/retry@v2
132+
with:
133+
max_attempts: 20
134+
timeout_seconds: 200
135+
shell: bash
136+
command: |
137+
SHERPA_ONNX_VERSION=v$(grep "SHERPA_ONNX_VERSION" ./CMakeLists.txt | cut -d " " -f 2 | cut -d '"' -f 2)
138+
139+
git config --global user.email "csukuangfj@gmail.com"
140+
git config --global user.name "Fangjun Kuang"
141+
142+
rm -rf huggingface
143+
export GIT_LFS_SKIP_SMUDGE=1
144+
export GIT_CLONE_PROTECTION_ACTIVE=false
145+
146+
git clone https://huggingface.co/spaces/k2-fsa/web-assembly-ten-vad-sherpa-onnx huggingface
147+
cd huggingface
148+
rm -fv *.js
149+
rm -fv *.data
150+
git fetch
151+
git pull
152+
git merge -m "merge remote" --ff origin main
153+
154+
cp -v ../sherpa-onnx-wasm-simd-${SHERPA_ONNX_VERSION}-ten-vad/* .
155+
156+
git status
157+
git lfs track "*.data"
158+
git lfs track "*.wasm"
159+
ls -lh
160+
161+
git add .
162+
git commit -m "update model"
163+
git push https://csukuangfj:$HF_TOKEN@huggingface.co/spaces/k2-fsa/web-assembly-ten-vad-sherpa-onnx main

‎nodejs-examples/test-vad-with-non-streaming-asr-moonshine.js

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,15 @@ function createRecognizer() {
2626
function createVad() {
2727
// please download silero_vad.onnx from
2828
// https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx
29+
//
30+
// please download ten-vad.onnx from
31+
// https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/ten-vad.onnx
32+
//
33+
// You only need one vad
34+
//
35+
// To use ten-vad.onnx, please set sileroVad.model to ''
36+
// and set tenVad.model to 'ten-vad.onnx'
37+
//
2938
const config = {
3039
sileroVad: {
3140
model: './silero_vad.onnx',
@@ -35,12 +44,22 @@ function createVad() {
3544
maxSpeechDuration: 5,
3645
windowSize: 512,
3746
},
47+
tenVad: {
48+
// model: './ten-vad.onnx',
49+
model: '',
50+
threshold: 0.5,
51+
minSpeechDuration: 0.25,
52+
minSilenceDuration: 0.5,
53+
maxSpeechDuration: 5,
54+
windowSize: 256,
55+
},
3856
sampleRate: 16000,
3957
debug: true,
4058
numThreads: 1,
4159
bufferSizeInSeconds: 60,
4260
};
4361

62+
4463
return sherpa_onnx.createVad(config);
4564
}
4665

@@ -60,7 +79,11 @@ if (wave.sampleRate != recognizer.config.featConfig.sampleRate) {
6079
console.log('Started')
6180
let start = Date.now();
6281

63-
const windowSize = vad.config.sileroVad.windowSize;
82+
let windowSize = vad.config.sileroVad.windowSize;
83+
if (vad.config.tenVad.model != '') {
84+
windowSize = vad.config.tenVad.windowSize;
85+
}
86+
6487
for (let i = 0; i < wave.samples.length; i += windowSize) {
6588
const thisWindow = wave.samples.subarray(i, i + windowSize);
6689
vad.acceptWaveform(thisWindow);

‎wasm/asr/assets/README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ rm -rf sherpa-onnx-streaming-paraformer-bilingual-zh-en
5959
6060
cd ../
6161
62-
sed -i.bak s/"type = 0"/"type = 1"/g ./sherpa-onnx.js
62+
sed -i.bak s/"type = 0"/"type = 1"/g ./sherpa-onnx-asr.js
6363
sed -i.bak s/Zipformer/Paraformer/g ./index.html
6464
6565
cd ../..

‎wasm/vad-asr/app-vad-asr.js

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,8 +69,6 @@ function fileExists(filename) {
6969
return exists;
7070
}
7171

72-
function createOfflineRecognizerSenseVoice() {}
73-
7472
function initOfflineRecognizer() {
7573
let config = {
7674
modelConfig: {

‎wasm/vad/CMakeLists.txt

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ if(NOT $ENV{SHERPA_ONNX_IS_USING_BUILD_WASM_SH})
22
message(FATAL_ERROR "Please use ./build-wasm-simd-vad.sh to build for wasm VAD")
33
endif()
44

5-
if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/assets/silero_vad.onnx")
5+
if(NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/assets/silero_vad.onnx" AND NOT EXISTS "${CMAKE_CURRENT_SOURCE_DIR}/assets/ten-vad.onnx" )
66
message(FATAL_ERROR "Please read ${CMAKE_CURRENT_SOURCE_DIR}/assets/README.md before you continue")
77
endif()
88

@@ -30,6 +30,7 @@ set(exported_functions
3030
SherpaOnnxVoiceActivityDetectorReset
3131
SherpaOnnxVoiceActivityDetectorFlush
3232
#
33+
SherpaOnnxFileExists
3334
)
3435
set(mangled_exported_functions)
3536
foreach(x IN LISTS exported_functions)

‎wasm/vad/app-vad.js

Lines changed: 81 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@
55
const startBtn = document.getElementById('startBtn');
66
const stopBtn = document.getElementById('stopBtn');
77
const clearBtn = document.getElementById('clearBtn');
8-
const hint = document.getElementById('hint');
98
const soundClips = document.getElementById('sound-clips');
109

1110
let textArea = document.getElementById('results');
@@ -43,19 +42,98 @@ function getDisplayResult() {
4342

4443

4544
Module = {};
45+
46+
// https://emscripten.org/docs/api_reference/module.html#Module.locateFile
47+
Module.locateFile = function(path, scriptDirectory = '') {
48+
console.log(`path: ${path}, scriptDirectory: ${scriptDirectory}`);
49+
return scriptDirectory + path;
50+
};
51+
52+
// https://emscripten.org/docs/api_reference/module.html#Module.locateFile
53+
Module.setStatus = function(status) {
54+
console.log(`status ${status}`);
55+
const statusElement = document.getElementById('status');
56+
if (status == 'Running...') {
57+
status = 'Model downloaded. Initializing vad...'
58+
}
59+
statusElement.textContent = status;
60+
if (status === '') {
61+
statusElement.style.display = 'none';
62+
// statusElement.parentNode.removeChild(statusElement);
63+
64+
document.querySelectorAll('.tab-content').forEach((tabContentElement) => {
65+
tabContentElement.classList.remove('loading');
66+
});
67+
} else {
68+
statusElement.style.display = 'block';
69+
document.querySelectorAll('.tab-content').forEach((tabContentElement) => {
70+
tabContentElement.classList.add('loading');
71+
});
72+
}
73+
};
74+
4675
Module.onRuntimeInitialized = function() {
4776
console.log('inited!');
48-
hint.innerText = 'Model loaded! Please click start';
4977

5078
startBtn.disabled = false;
5179

52-
vad = createVad(Module);
80+
initVad();
5381
console.log('vad is created!', vad);
5482

5583
buffer = new CircularBuffer(30 * 16000, Module);
5684
console.log('CircularBuffer is created!', buffer);
5785
};
5886

87+
function fileExists(filename) {
88+
const filenameLen = Module.lengthBytesUTF8(filename) + 1;
89+
const buffer = Module._malloc(filenameLen);
90+
Module.stringToUTF8(filename, buffer, filenameLen);
91+
92+
let exists = Module._SherpaOnnxFileExists(buffer);
93+
94+
Module._free(buffer);
95+
96+
return exists;
97+
}
98+
99+
function initVad() {
100+
const sileroVad = {
101+
model: '',
102+
threshold: 0.50,
103+
minSilenceDuration: 0.50,
104+
minSpeechDuration: 0.25,
105+
maxSpeechDuration: 20,
106+
windowSize: 512,
107+
};
108+
109+
const tenVad = {
110+
model: '',
111+
threshold: 0.50,
112+
minSilenceDuration: 0.50,
113+
minSpeechDuration: 0.25,
114+
maxSpeechDuration: 20,
115+
windowSize: 256,
116+
};
117+
118+
let config = {
119+
sileroVad: sileroVad,
120+
tenVad: tenVad,
121+
sampleRate: 16000,
122+
numThreads: 1,
123+
provider: 'cpu',
124+
debug: 1,
125+
bufferSizeInSeconds: 30,
126+
};
127+
128+
if (fileExists('silero_vad.onnx') == 1) {
129+
config.sileroVad.model = 'silero_vad.onnx'
130+
} else if (fileExists('ten-vad.onnx') == 1) {
131+
config.tenVad.model = 'ten-vad.onnx'
132+
}
133+
134+
vad = createVad(Module, config);
135+
}
136+
59137
let audioCtx;
60138
let mediaStream;
61139

‎wasm/vad/assets/README.md

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,32 @@
11
# Introduction
22

3+
## Use silero-vad
4+
35
Please download
46
https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx
57
and put `silero_vad.onnx` into the current directory, i.e., `wasm/vad/assets`.
68

79
You can find example build script at
810
https://github.com/k2-fsa/sherpa-onnx/blob/master/.github/workflows/wasm-simd-hf-space-silero-vad.yaml
11+
12+
```
13+
cd /path/to/sherpa-onnx/wasm/vad/assets
14+
wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/silero_vad.onnx
15+
```
16+
17+
## Use ten-vad
18+
19+
Please download
20+
https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/ten-vad.onnx
21+
and put `ten-vad.onnx` into the current directory, i.e., `wasm/vad/assets`.
22+
23+
You can find example build script at
24+
https://github.com/k2-fsa/sherpa-onnx/blob/master/.github/workflows/wasm-simd-hf-space-ten-vad.yaml
25+
26+
```
27+
cd /path/to/sherpa-onnx/wasm/vad/assets
28+
wget https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/ten-vad.onnx
29+
cd ..
30+
sed -i.bak "s|.*(with <a .*| (with <a href="https://github.com/TEN-framework/ten-vad">ten-vad</a>)|" ./index.html
31+
32+
```

0 commit comments

Comments
 (0)