瀏覽代碼

Update streaming README and add seamless_s2st.py (#186)

* Update streaming README

* Add seamless_s2st.py
Abinesh Ramakrishnan 1 年之前
父節點
當前提交
004e89b0ee

+ 1 - 1
src/seamless_communication/cli/streaming/README.md

@@ -22,7 +22,7 @@ Note: The `--ref-field` can be used to specify the name of the reference column
 Set the task to `asr` for evaluating the automatic speech recognition part of the SeamlessStreaming model. Make sure to pass the source language as the `--tgt-lang` arg.
 Set the task to `asr` for evaluating the automatic speech recognition part of the SeamlessStreaming model. Make sure to pass the source language as the `--tgt-lang` arg.
 
 
 ```bash
 ```bash
-streaming_evaluate --task s2tt --data-file <path_to_data_tsv_file> --audio-root-dir <path_to_audio_root_directory> --output <path_to_evaluation_output_directory> --tgt-lang <3_letter_source_lang_code> 
+streaming_evaluate --task asr --data-file <path_to_data_tsv_file> --audio-root-dir <path_to_audio_root_directory> --output <path_to_evaluation_output_directory> --tgt-lang <3_letter_source_lang_code> 
 ```
 ```
 
 
 ### S2ST:
 ### S2ST:

+ 50 - 0
src/seamless_communication/streaming/agents/seamless_s2st.py

@@ -0,0 +1,50 @@
+# Copyright (c) Meta Platforms, Inc. and affiliates
+# All rights reserved.
+#
+# This source code is licensed under the license found in the
+# LICENSE file in the root directory of this source tree.
+
+
+from seamless_communication.streaming.agents.detokenizer import UnitYDetokenizerAgent
+from seamless_communication.streaming.agents.offline_w2v_bert_encoder import (
+    OfflineWav2VecBertEncoderAgent,
+)
+from seamless_communication.streaming.agents.online_feature_extractor import (
+    OnlineFeatureExtractorAgent,
+)
+from seamless_communication.streaming.agents.online_text_decoder import (
+    UnitYMMATextDecoderAgent,
+)
+from seamless_communication.streaming.agents.online_unit_decoder import (
+    NARUnitYUnitDecoderAgent,
+)
+from seamless_communication.streaming.agents.pretssel_vocoder import (
+    PretsselVocoderAgent,
+)
+from seamless_communication.streaming.agents.silero_vad import SileroVADAgent
+from seamless_communication.streaming.agents.unity_pipeline import (
+    UnitYAgentPipeline,
+    UnitYAgentTreePipeline,
+)
+
+
+class SeamlessS2STAgent(UnitYAgentPipeline):
+    pipeline = [
+        OnlineFeatureExtractorAgent,
+        OfflineWav2VecBertEncoderAgent,
+        UnitYMMATextDecoderAgent,
+        NARUnitYUnitDecoderAgent,
+        PretsselVocoderAgent,
+    ]
+
+
+class SeamlessS2STJointVADAgent(UnitYAgentTreePipeline):
+    pipeline = {
+        SileroVADAgent: [OnlineFeatureExtractorAgent],
+        OnlineFeatureExtractorAgent: [OfflineWav2VecBertEncoderAgent],
+        OfflineWav2VecBertEncoderAgent: [UnitYMMATextDecoderAgent],
+        UnitYMMATextDecoderAgent: [UnitYDetokenizerAgent, NARUnitYUnitDecoderAgent],
+        UnitYDetokenizerAgent: [],
+        NARUnitYUnitDecoderAgent: [PretsselVocoderAgent],
+        PretsselVocoderAgent: [],
+    }