Add Java and Kotlin API for NeMo Canary models (#2359)

csukuangfj · web-flow · commit 103e93d9f648 · 2025-07-08T13:45:26.000+08:00
Add support for the NeMo Canary model in both Java and Kotlin APIs, wiring it through
JNI and updating examples and CI.

- Introduce OfflineCanaryModelConfig in Kotlin and Java with builder patterns
- Extend OfflineRecognizer to accept and apply the new canary config via setConfig
- Update JNI binding (GetOfflineConfig) and getOfflineModelConfig mapping (type 32), 
   plus examples and CI workflows
diff --git a/.github/workflows/run-java-test.yaml b/.github/workflows/run-java-test.yaml
@@ -117,6 +117,13 @@ jobs:
           cd ./java-api-examples
           ./run-version-test.sh
 
+      - name:  Run java test (Nemo Canary)
+        shell: bash
+        run: |
+          cd ./java-api-examples
+          ./run-non-streaming-decode-file-nemo-canary.sh
+          rm -rf sherpa-onnx-nemo-*
+
       - name:  Run java test (Non-streaming SenseVoice with homophone replacer)
         shell: bash
         run: |
diff --git a/java-api-examples/NonStreamingDecodeFileNemoCanary.java b/java-api-examples/NonStreamingDecodeFileNemoCanary.java
@@ -0,0 +1,56 @@
+// Copyright 2024 Xiaomi Corporation
+
+// This file shows how to use an offline NeMo Canary model, i.e.,
+// non-streaming NeMo Canary model, to decode files.
+import com.k2fsa.sherpa.onnx.*;
+
+public class NonStreamingDecodeFileNemoCanary {
+  public static void main(String[] args) {
+    // please refer to
+    // https://k2-fsa.github.io/sherpa/onnx/nemo/canary.html
+    // to download model files
+    String encoder = "./sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/encoder.int8.onnx";
+    String decoder = "./sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/decoder.int8.onnx";
+    String tokens = "./sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/tokens.txt";
+
+    String waveFilename = "./sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/test_wavs/en.wav";
+
+    WaveReader reader = new WaveReader(waveFilename);
+
+    OfflineCanaryModelConfig canary =
+        OfflineCanaryModelConfig.builder()
+            .setEncoder(encoder)
+            .setDecoder(decoder)
+            .setSrcLang("en")
+            .setTgtLang("en")
+            .setUsePnc(true)
+            .build();
+
+    OfflineModelConfig modelConfig =
+        OfflineModelConfig.builder()
+            .setCanary(canary)
+            .setTokens(tokens)
+            .setNumThreads(1)
+            .setDebug(true)
+            .build();
+
+    OfflineRecognizerConfig config =
+        OfflineRecognizerConfig.builder()
+            .setOfflineModelConfig(modelConfig)
+            .setDecodingMethod("greedy_search")
+            .build();
+
+    OfflineRecognizer recognizer = new OfflineRecognizer(config);
+    OfflineStream stream = recognizer.createStream();
+    stream.acceptWaveform(reader.getSamples(), reader.getSampleRate());
+
+    recognizer.decode(stream);
+
+    String text = recognizer.getResult(stream).getText();
+
+    System.out.printf("filename:%s\nresult(English):%s\n", waveFilename, text);
+
+    stream.release();
+    recognizer.release();
+  }
+}
diff --git a/java-api-examples/README.md b/java-api-examples/README.md
@@ -24,11 +24,18 @@ This directory contains examples for the JAVA API of sherpa-onnx.
 
 ```bash
 ./run-non-streaming-decode-file-dolphin-ctc.sh
+./run-non-streaming-decode-file-fire-red-asr.sh
+./run-non-streaming-decode-file-moonshine.sh
+./run-non-streaming-decode-file-nemo-canary.sh
+./run-non-streaming-decode-file-nemo.sh
 ./run-non-streaming-decode-file-paraformer.sh
 ./run-non-streaming-decode-file-sense-voice.sh
+./run-non-streaming-decode-file-tele-speech-ctc.sh
+./run-non-streaming-decode-file-transducer-hotwords.sh
 ./run-non-streaming-decode-file-transducer.sh
+./run-non-streaming-decode-file-whisper-multiple.sh
 ./run-non-streaming-decode-file-whisper.sh
-./run-non-streaming-decode-file-nemo.sh
+./run-non-streaming-decode-file-zipformer-ctc.sh
 ```
 
 ## Non-Streaming Speech recognition with homophone replacer
diff --git a/java-api-examples/run-non-streaming-decode-file-nemo-canary.sh b/java-api-examples/run-non-streaming-decode-file-nemo-canary.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env bash
+
+set -ex
+
+if [[ ! -f ../build/lib/libsherpa-onnx-jni.dylib  && ! -f ../build/lib/libsherpa-onnx-jni.so ]]; then
+  mkdir -p ../build
+  pushd ../build
+  cmake \
+    -DSHERPA_ONNX_ENABLE_PYTHON=OFF \
+    -DSHERPA_ONNX_ENABLE_TESTS=OFF \
+    -DSHERPA_ONNX_ENABLE_CHECK=OFF \
+    -DBUILD_SHARED_LIBS=ON \
+    -DSHERPA_ONNX_ENABLE_PORTAUDIO=OFF \
+    -DSHERPA_ONNX_ENABLE_JNI=ON \
+    ..
+
+  make -j4
+  ls -lh lib
+  popd
+fi
+
+if [ ! -f ../sherpa-onnx/java-api/build/sherpa-onnx.jar ]; then
+  pushd ../sherpa-onnx/java-api
+  make
+  popd
+fi
+
+if [ ! -f sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/encoder.int8.onnx ]; then
+  curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2
+  tar xvf sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2
+  rm sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2
+fi
+
+java \
+  -Djava.library.path=$PWD/../build/lib \
+  -cp ../sherpa-onnx/java-api/build/sherpa-onnx.jar \
+  NonStreamingDecodeFileNemoCanary.java
diff --git a/kotlin-api-examples/run.sh b/kotlin-api-examples/run.sh
@@ -455,8 +455,31 @@ function testOfflineSenseVoiceWithHr() {
   ls -lh $out_filename
   java -Djava.library.path=../build/lib -jar $out_filename
 }
-testVersion
 
+function testOfflineNeMoCanary() {
+  if [ ! -f sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/encoder.int8.onnx ]; then
+    curl -SL -O https://github.com/k2-fsa/sherpa-onnx/releases/download/asr-models/sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2
+    tar xvf sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2
+    rm sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8.tar.bz2
+  fi
+
+  out_filename=test_offline_nemo_canary.jar
+  kotlinc-jvm -include-runtime -d $out_filename \
+    test_offline_nemo_canary.kt \
+    FeatureConfig.kt \
+    HomophoneReplacerConfig.kt \
+    OfflineRecognizer.kt \
+    OfflineStream.kt \
+    WaveReader.kt \
+    faked-asset-manager.kt
+
+  ls -lh $out_filename
+  java -Djava.library.path=../build/lib -jar $out_filename
+}
+
+# testVersion
+
+testOfflineNeMoCanary
 testOfflineSenseVoiceWithHr
 testOfflineSpeechDenoiser
 testOfflineSpeakerDiarization
diff --git a/kotlin-api-examples/test_offline_nemo_canary.kt b/kotlin-api-examples/test_offline_nemo_canary.kt
@@ -0,0 +1,48 @@
+package com.k2fsa.sherpa.onnx
+
+fun main() {
+  val recognizer = createOfflineRecognizer()
+  val waveFilename = "./sherpa-onnx-nemo-canary-180m-flash-en-es-de-fr-int8/test_wavs/en.wav"
+
+  val objArray = WaveReader.readWaveFromFile(
+      filename = waveFilename,
+  )
+  val samples: FloatArray = objArray[0] as FloatArray
+  val sampleRate: Int = objArray[1] as Int
+
+  var stream = recognizer.createStream()
+  stream.acceptWaveform(samples, sampleRate=sampleRate)
+  recognizer.decode(stream)
+
+  var result = recognizer.getResult(stream)
+  println("English: $result")
+
+  stream.release()
+
+  // now output text in German
+  val config = recognizer.config.copy(modelConfig=recognizer.config.modelConfig.copy(
+    canary=recognizer.config.modelConfig.canary.copy(
+      tgtLang="de"
+    )
+  ))
+  recognizer.setConfig(config)
+
+  stream = recognizer.createStream()
+  stream.acceptWaveform(samples, sampleRate=sampleRate)
+  recognizer.decode(stream)
+
+  result = recognizer.getResult(stream)
+  println("German: $result")
+
+  stream.release()
+  recognizer.release()
+}
+
+
+fun createOfflineRecognizer(): OfflineRecognizer {
+  val config = OfflineRecognizerConfig(
+      modelConfig = getOfflineModelConfig(type = 32)!!,
+  )
+
+  return OfflineRecognizer(config = config)
+}
diff --git a/sherpa-onnx/java-api/Makefile b/sherpa-onnx/java-api/Makefile
@@ -34,6 +34,7 @@ java_files += OfflineFireRedAsrModelConfig.java
 java_files += OfflineMoonshineModelConfig.java
 java_files += OfflineNemoEncDecCtcModelConfig.java
 java_files += OfflineZipformerCtcModelConfig.java
+java_files += OfflineCanaryModelConfig.java
 java_files += OfflineSenseVoiceModelConfig.java
 java_files += OfflineDolphinModelConfig.java
 java_files += OfflineModelConfig.java
diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineCanaryModelConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineCanaryModelConfig.java
@@ -0,0 +1,80 @@
+// Copyright 2025 Xiaomi Corporation
+
+package com.k2fsa.sherpa.onnx;
+
+public class OfflineCanaryModelConfig {
+    private final String encoder;
+    private final String decoder;
+    private final String srcLang;
+    private final String tgtLang;
+    private final boolean usePnc;
+
+    private OfflineCanaryModelConfig(Builder builder) {
+        this.encoder = builder.encoder;
+        this.decoder = builder.decoder;
+        this.srcLang = builder.srcLang;
+        this.tgtLang = builder.tgtLang;
+        this.usePnc = builder.usePnc;
+    }
+
+    public static Builder builder() {
+        return new Builder();
+    }
+
+    public String getEncoder() {
+        return encoder;
+    }
+
+    public String getDecoder() {
+        return decoder;
+    }
+
+    public String getSrcLang() {
+        return srcLang;
+    }
+
+    public String getTgtLang() {
+        return tgtLang;
+    }
+
+    public boolean isUsePnc() {
+        return usePnc;
+    }
+
+    public static class Builder {
+        private String encoder = "";
+        private String decoder = "";
+        private String srcLang = "en";
+        private String tgtLang = "en";
+        private boolean usePnc = true;
+
+        public OfflineCanaryModelConfig build() {
+            return new OfflineCanaryModelConfig(this);
+        }
+
+        public Builder setEncoder(String encoder) {
+            this.encoder = encoder;
+            return this;
+        }
+
+        public Builder setDecoder(String decoder) {
+            this.decoder = decoder;
+            return this;
+        }
+
+        public Builder setSrcLang(String srcLang) {
+            this.srcLang = srcLang;
+            return this;
+        }
+
+        public Builder setTgtLang(String tgtLang) {
+            this.tgtLang = tgtLang;
+            return this;
+        }
+
+        public Builder setUsePnc(boolean usePnc) {
+            this.usePnc = usePnc;
+            return this;
+        }
+    }
+}
diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineModelConfig.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineModelConfig.java
@@ -12,6 +12,7 @@ public class OfflineModelConfig {
     private final OfflineSenseVoiceModelConfig senseVoice;
     private final OfflineDolphinModelConfig dolphin;
     private final OfflineZipformerCtcModelConfig zipformerCtc;
+    private final OfflineCanaryModelConfig canary;
     private final String teleSpeech;
     private final String tokens;
     private final int numThreads;
@@ -30,6 +31,7 @@ private OfflineModelConfig(Builder builder) {
         this.moonshine = builder.moonshine;
         this.nemo = builder.nemo;
         this.zipformerCtc = builder.zipformerCtc;
+        this.canary = builder.canary;
         this.senseVoice = builder.senseVoice;
         this.dolphin = builder.dolphin;
         this.teleSpeech = builder.teleSpeech;
@@ -78,6 +80,10 @@ public OfflineZipformerCtcModelConfig getZipformerCtc() {
         return zipformerCtc;
     }
 
+    public OfflineCanaryModelConfig getCanary() {
+        return canary;
+    }
+
     public String getTokens() {
         return tokens;
     }
@@ -120,6 +126,7 @@ public static class Builder {
         private OfflineSenseVoiceModelConfig senseVoice = OfflineSenseVoiceModelConfig.builder().build();
         private OfflineDolphinModelConfig dolphin = OfflineDolphinModelConfig.builder().build();
         private OfflineZipformerCtcModelConfig zipformerCtc = OfflineZipformerCtcModelConfig.builder().build();
+        private OfflineCanaryModelConfig canary = OfflineCanaryModelConfig.builder().build();
         private String teleSpeech = "";
         private String tokens = "";
         private int numThreads = 1;
@@ -158,6 +165,11 @@ public Builder setZipformerCtc(OfflineZipformerCtcModelConfig zipformerCtc) {
             return this;
         }
 
+        public Builder setCanary(OfflineCanaryModelConfig canary) {
+            this.canary = canary;
+            return this;
+        }
+
         public Builder setTeleSpeech(String teleSpeech) {
             this.teleSpeech = teleSpeech;
             return this;
diff --git a/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineRecognizer.java b/sherpa-onnx/java-api/src/com/k2fsa/sherpa/onnx/OfflineRecognizer.java
@@ -4,10 +4,22 @@
 
 public class OfflineRecognizer {
     private long ptr = 0;
+    private final OfflineRecognizerConfig config;
 
     public OfflineRecognizer(OfflineRecognizerConfig config) {
         LibraryLoader.maybeLoad();
         ptr = newFromFile(config);
+
+        this.config = config;
+    }
+
+    public void setConfig(OfflineRecognizerConfig config) {
+        setConfig(ptr, config);
+        // we don't update this.config
+    }
+
+    public OfflineRecognizerConfig getConfig() {
+        return config;
     }
 
     public void decode(OfflineStream s) {
@@ -60,6 +72,8 @@ public OfflineRecognizerResult getResult(OfflineStream s) {
 
     private native void decode(long ptr, long streamPtr);
 
+    private native void setConfig(long ptr, OfflineRecognizerConfig config);
+
     private native void decodeStreams(long ptr, long[] streamPtrs);
 
     private native Object[] getResult(long streamPtr);
diff --git a/sherpa-onnx/jni/offline-recognizer.cc b/sherpa-onnx/jni/offline-recognizer.cc
diff --git a/sherpa-onnx/kotlin-api/OfflineRecognizer.kt b/sherpa-onnx/kotlin-api/OfflineRecognizer.kt
