Interactive only unity.cpp & Fix warnings (#334)

* Sync unity.cpp

Co-authored-by: Tuan Tran <1254753+antoine-tran@users.noreply.github.com>
Co-authored-by: Guillaume Wenzek <5920036+gwenzek@users.noreply.github.com>
Co-authored-by: Ning <7022920+cndn@users.noreply.github.com>

* address comments

* Interactive-only unity.cpp with S2TT+T2TT

* Update README.md

* Fix all warnings

---------

Co-authored-by: Tuan Tran <1254753+antoine-tran@users.noreply.github.com>
Co-authored-by: Guillaume Wenzek <5920036+gwenzek@users.noreply.github.com>
Co-authored-by: Ning <7022920+cndn@users.noreply.github.com>

ggml/README.md

@@ -6,7 +6,7 @@
 The project is still active in development. Contributions are welcome!
 
 ## Build
-To build the interactive console for S2TT & ASR, 
+To build the interactive console for S2TT & ASR & T2TT, 
 ```
 
 cd seamless_communication/ggml
@@ -22,11 +22,22 @@ make -j4 unity # Interactive Console
 For more build commands see [Makefile](Makefile). 
 
 ## CLI usage
+### S2TT
 Command to launch an interactive console for S2TT & ASR, note that the model already includes vocabulary needed to detokenize. 
 ```
 OPENBLAS_NUM_THREADS=8 ./bin/unity --model seamlessM4T_medium.ggml
 ```
-In the console, enter the path of local waveform file and target language, separated by space. Note that the first run would include some “warm up” time so could be slow. 
+In the console, enter "wav_file tgt_lang" - the path of local waveform file and target language, separated by space. Note that the first run would include some “warm up” time so could be slow. 
+
+### T2TT
+Launching command:
+```
+OPENBLAS_NUM_THREADS=8 ./bin/unity --model nllb-200_dense_1b.ggml --text
+```
+In the console, enter "input_text tgt_lang" - input text and target langauge, separated by space. Note that the language code should align with [NLLB BCP-47 code](https://github.com/facebookresearch/flores/blob/main/flores200/README.md#languages-in-flores-200), NOT 3-letter language code as S2TT task with Seamless. Unifying this is on todo list. 
+
+
+### Model downloads 
 
 Converted ggml models could be downloaded from 
 |SeamlessM4T_large | SeamlessM4T_medium | NLLB_dense_1b | NLLB_distill_600m |

ggml/examples/unity/fairseq2.cpp

@@ -25,7 +25,7 @@ ggml_tensor* ggml_detach(ggml_tensor* a) {
 // when we read garbage data.
 // It also prints memory usage information, which is useful to
 #define DEBUG_MEM_USAGE DEBUG
-size_t MB = 1024 * 1024;
+std::size_t MB = 1024 * 1024;
 
 void printf_mem_usage(ggml_context* ctx, std::string name) {
 #if DEBUG_MEM_USAGE
@@ -861,9 +861,9 @@ ggml_tensor* ggml_slice(
 
 
     ne[axis] = end - start;
-    size_t offset = a->nb[axis] * start;
+    std::size_t offset = a->nb[axis] * start;
 
-    size_t* nb = a->nb;
+    std::size_t* nb = a->nb;
     ggml_tensor* result = ggml_view_4d(ctx, a, ne[0], ne[1], ne[2], ne[3], nb[1], nb[2], nb[3], offset);
     ggml_format_name(result, "%s [(%d)%ld:%ld]", a->name, axis, start, end);
     result->n_dims = a->n_dims;
@@ -886,8 +886,8 @@ ggml_tensor* ggml_select(
 
     std::copy(a->ne + axis + 1, a->ne + GGML_MAX_DIMS, ne + axis);
 
-    size_t offset = a->nb[axis] * index;
-    size_t* nb = a->nb;
+    std::size_t offset = a->nb[axis] * index;
+    std::size_t* nb = a->nb;
     GGML_ASSERT(GGML_MAX_DIMS == 4);
     ggml_tensor* result = ggml_view_3d(ctx, a, ne[0], ne[1], ne[2], nb[1], nb[2], offset);
     ggml_format_name(result, "%s [(%d)%ld]", a->name, axis, index);
@@ -1216,18 +1216,18 @@ void _bootstrap_seqs_and_scores(
     full_seqs->type = GGML_TYPE_I32;
     job.prefix_seq->type = GGML_TYPE_I32;
     // For LID
-    for (size_t i = 0; i < lang_ids.size(); ++i) {
+    for (std::size_t i = 0; i < lang_ids.size(); ++i) {
         ggml_set_f32_1d(lid_scores, i, std::exp(ggml_get_f32_1d(lprobs, lang_ids[i])));
     }
 
     // Fetch scores of next steps from "lprobs"
     float p_score = 0;
     for (int i = 1; i < prefix_seq_len; ++i) {
-        int p;
+        int p = 0;
         if (ggml_get_i32_1d(job.prefix_seq, i) == model.vocab.token_to_id["<unk>"]) {
             // If tgt_lang is unk, use the most probable lang tag predicted by model
             int max_value = std::numeric_limits<float>::min();
-            for (int j = 0; j < lang_ids.size(); j++) {
+            for (std::size_t j = 0; j < lang_ids.size(); j++) {
                 if(ggml_get_f32_1d(lprobs, lang_ids[j]) > max_value) {
                     max_value = ggml_get_f32_1d(lprobs, lang_ids[j]);
                     p = lang_ids[j];
@@ -1273,7 +1273,7 @@ void _tweak_lprobs(const SequenceGeneratorJob& job, ggml_tensor* lprobs, int ste
     // Do not allow EOS before reaching the minimum sequence length.
     if (step_nr < job.opts.min_seq_len) {
         // lprobs[:, :, self.eos_idx] = -INFINITY;
-        for (size_t i = 0; i < beam_size; ++i)
+        for (std::size_t i = 0; i < beam_size; ++i)
             ggml_set_f32_1d(lprobs, vocab_size * i + eos_idx, -INFINITY);
     }
 
@@ -1281,8 +1281,8 @@ void _tweak_lprobs(const SequenceGeneratorJob& job, ggml_tensor* lprobs, int ste
     if (step_nr == max_seq_len - 2) {
         // lprobs[:, :, : self.eos_idx]       = -torch.inf
         // lprobs[:, :,   self.eos_idx + 1 :] = -torch.inf
-        for (size_t b = 0; b < beam_size; ++b) {
-            size_t t = 0;
+        for (std::size_t b = 0; b < beam_size; ++b) {
+            std::size_t t = 0;
             for (t = 0; t < eos_idx; ++t)
                 ggml_set_f32_1d(lprobs, vocab_size * b + t, -INFINITY);
             for (t = eos_idx + 1; t < vocab_size; ++t)
@@ -1292,14 +1292,14 @@ void _tweak_lprobs(const SequenceGeneratorJob& job, ggml_tensor* lprobs, int ste
 
     // Never allow PAD.
     std::size_t pad_idx = job.pad_idx;
-    for (size_t i = 0; i < beam_size; ++i)
+    for (std::size_t i = 0; i < beam_size; ++i)
         ggml_set_f32_1d(lprobs, vocab_size * i + pad_idx, -INFINITY);
 
     // Apply UNK penalty.
     if (job.unk_idx >= 0 && job.opts.unk_penalty != 0) {
         // lprobs[:, :, self.unk_idx] -= self.opts.unk_penalty
         auto lprobs_raw = ggml_get_data_f32(lprobs);
-        for (size_t i = 0; i < beam_size; ++i)
+        for (std::size_t i = 0; i < beam_size; ++i)
             lprobs_raw[vocab_size * i + job.unk_idx] -= job.opts.unk_penalty;
     }
 }
@@ -1354,7 +1354,7 @@ void _finalize_hypothesis(
 
 ggml_context* ctx_from_buffer(std::vector<uint8_t>& buffer) {
     return ggml_init({
-        /*.mem_size   =*/ static_cast<int64_t>(buffer.capacity()),
+        /*.mem_size   =*/ static_cast<std::size_t>(buffer.capacity()),
         /*.mem_buffer =*/ buffer.data(),
         /*.no_alloc   =*/ false,
     });
@@ -1404,7 +1404,7 @@ extern "C" Hypothesis* generate_sequence(
         std::sort(lang_ids.begin(), lang_ids.end());
     }
     ggml_tensor* embed = model.tensors["text_decoder_frontend.embed.weight"];
-    size_t vocab_size = embed->ne[1];
+    std::size_t vocab_size = embed->ne[1];
     std::size_t beam_size = job.opts.beam_size;
     ggml_detach(encoder_output);
     int source_seq_len = encoder_output->ne[1];
@@ -1438,7 +1438,7 @@ extern "C" Hypothesis* generate_sequence(
     ggml_context* step_ctx = ctx_from_buffer(local_bufs[start_step % 2]);
     GGML_ASSERT(step_ctx != search_ctx);
     model.enc_kv_cache_ctx = search_ctx;
-    ggml_tensor* lid_scores;
+    ggml_tensor* lid_scores = ggml_new_tensor_1d(result_ctx, GGML_TYPE_F32, 1); // Dummy initialization to get rid of warnings
     if (lang_ids.size()) {
         lid_scores = ggml_new_tensor_1d(result_ctx, GGML_TYPE_F32, lang_ids.size());
     } 
@@ -1463,20 +1463,19 @@ extern "C" Hypothesis* generate_sequence(
     for (int step_nr = start_step; step_nr < max_seq_len - 1; ++step_nr) {
         model.ctx = step_ctx;
         ggml_set_no_alloc(step_ctx, true); // Use allocr for the model forward pass
-        float max_lprob;
-        int p;
+        int p = 0;
         if (step_nr == start_step) {
             // Find the most probable lang_tok and assign it to all beams, when prefix_seq[1] is <unk>
             if (lang_ids.size() && ggml_get_i32_1d(job.prefix_seq, 1) == model.vocab.token_to_id["<unk>"]) {
                 float max_lprob = std::numeric_limits<float>::min();
-                for(int j = 0; j < lang_ids.size(); j++) {
+                for(std::size_t j = 0; j < lang_ids.size(); j++) {
                     auto val = ggml_get_f32_1d(lid_scores, j);
                     if (val > max_lprob) {
                         max_lprob = val;
                         p = lang_ids[j];
                     }
                 }
-                for (int k = 0; k < beam_size; k++) {
+                for (std::size_t k = 0; k < beam_size; k++) {
                     ggml_set_i32_1d(seqs, k * vocab_size + step_nr, p);
                 }
             }
@@ -1503,7 +1502,7 @@ extern "C" Hypothesis* generate_sequence(
         // TODO: use ggml properly compute the tweaks
         struct ggml_cgraph * gf = ggml_new_graph(step_ctx);
         ggml_build_forward_expand(gf, lprobs);
-        size_t fwd_mem = ggml_allocr_alloc_graph(step_alloc, gf);
+        std::size_t fwd_mem = ggml_allocr_alloc_graph(step_alloc, gf);
         GGML_UNUSED(fwd_mem);
         ggml_graph_compute_with_ctx(step_ctx, gf, n_threads);
         ggml_detach(lprobs);
@@ -1631,14 +1630,14 @@ struct llm_symbol {
     index prev;
     index next;
     const char * text;
-    size_t n;
+    std::size_t n;
     llama_vocab::id id;
 };
 
 static_assert(std::is_trivially_copyable<llm_symbol>::value, "llm_symbol is not trivially copyable");
 
-static size_t utf8_len(char src) {
-    const size_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 };
+static std::size_t utf8_len(char src) {
+    const std::size_t lookup[] = { 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 2, 2, 3, 4 };
     uint8_t highbits = static_cast<uint8_t>(src) >> 4;
     return lookup[highbits];
 }
@@ -1654,7 +1653,7 @@ struct llm_bigram_spm {
     llm_symbol::index left;
     llm_symbol::index right;
     float score;
-    size_t size;
+    std::size_t size;
     llama_vocab::id id;
 };
 
@@ -1666,7 +1665,7 @@ struct llm_tokenizer_spm {
 
         // split string into utf8 chars
         int index = 0;
-        size_t offs = 0;
+        std::size_t offs = 0;
         // This is kind of annoying, but needed because with SPM,
         // characters following a space have a special meaning.
         // And the algorithm rely on substrings to do the lookups.
@@ -1675,8 +1674,8 @@ struct llm_tokenizer_spm {
         if (need_extra_space) text = " " + text;
 
         while (offs < text.size()) {
-            size_t len = utf8_len(text[offs]);
-            size_t n = std::min(len, text.size() - offs);
+            std::size_t len = utf8_len(text[offs]);
+            std::size_t n = std::min(len, text.size() - offs);
 
             auto token = vocab.token_to_id.find(std::string(text, offs, n));
             llama_vocab::id id = token == vocab.token_to_id.end() ? unk_idx : token->second;
@@ -1693,7 +1692,7 @@ struct llm_tokenizer_spm {
         }
 
         // seed the work queue with all possible 2-character tokens.
-        for (size_t i = 1; i < symbols.size(); ++i) {
+        for (std::size_t i = 1; i < symbols.size(); ++i) {
             try_add_bigram(i - 1, i);
         }
 
@@ -1757,7 +1756,7 @@ private:
         }
 
         llama_vocab::id id = token->second;
-        if (static_cast<size_t>(id) >= vocab.id_to_token.size()) {
+        if (static_cast<std::size_t>(id) >= vocab.id_to_token.size()) {
             return;
         }
 

ggml/examples/unity/model_loader.cpp

@@ -47,7 +47,7 @@ model_loader::load_model_weights(fairseq2_model &model, std::ifstream &fin)
     // Note this require changing the on disk format
     bool as_float32 = true;
     struct ggml_init_params params = {
-        /*.mem_size   =*/ f32_tensor_size + (num_tensor + 1) * (int64_t)ggml_tensor_overhead(),
+        /*.mem_size   =*/ static_cast<size_t>(f32_tensor_size + (num_tensor + 1) * (int64_t)ggml_tensor_overhead()),
         /*.mem_buffer =*/ NULL,
         /*.no_alloc   =*/ false,
     };
@@ -144,7 +144,7 @@ void model_loader::load_vocab(llama_vocab& vocab, std::ifstream &fin)
     std::string packed_vocab = get_name(fin);
     std::int64_t ctx_size = vocab_size * sizeof(float) + vocab_size + 2 * ggml_tensor_overhead();
     ctx_size *= 2;
-    ggml_context* ctx = ggml_init(ggml_init_params{ctx_size, nullptr, false});
+    ggml_context* ctx = ggml_init(ggml_init_params{static_cast<size_t>(ctx_size), nullptr, false});
     ggml_tensor* lengths_tensor = load_tensor_value(fin, ctx, true);
     std::int8_t* lengths = (std::int8_t*)lengths_tensor->data;
     ggml_tensor* scores_tensor = load_tensor_value(fin, ctx, true);

ggml/examples/unity/unity.cpp

@@ -14,9 +14,6 @@
 struct unity_params {
     int32_t n_threads = std::min(4, (int32_t) std::thread::hardware_concurrency());
     std::string model = "seamlessM4T_medium.ggml"; // model path
-    std::string input_text = "";
-    std::string tgt_lang = "eng";
-    std::vector<std::string> files = {};
     bool text = false;
     SequenceGeneratorOptions opts = {
         /*beam_size*/ 5,
@@ -39,14 +36,11 @@ void unity_print_usage(int /*argc*/, char ** argv, const unity_params & params)
     fprintf(stderr, "\n");
     fprintf(stderr, "options:\n");
     fprintf(stderr, "  -h, --help            show this help message and exit\n");
-    fprintf(stderr, "  -i, --input           Input text for the text-2-text translation\n");
-    fprintf(stderr, "  -l, --tgt-lang        Target translation lang (default: %s\n", params.tgt_lang);
-
     fprintf(stderr, "  -t N, --threads N     number of threads to use during computation (default: %d)\n", params.n_threads);
     fprintf(stderr, "  -v, --verbose         Print out word level confidence score and LID score (default: off)");
     fprintf(stderr, "  -m FNAME, --model FNAME\n");
     fprintf(stderr, "                        model path (default: %s)\n", params.model.c_str());
-    fprintf(stderr, "  --text                text output\n");
+    fprintf(stderr, "  --text                text-to-text translation (default is speech-to-text without this option on)\n");
     fprintf(stderr, "  --beam-size           beam size (default: %d)\n", params.opts.beam_size);
     fprintf(stderr, "  -M, --mem             memory buffer, increase for long inputs (default: %d)\n", params.opts.mem_mb);
     fprintf(stderr, " --max-audio max duration of audio in seconds (default: %d)\n", params.max_audio_s);
@@ -73,10 +67,6 @@ bool unity_params_parse(int argc, char ** argv, unity_params & params) {
             params.n_threads = std::stoi(get_next_arg(i, argc, argv, arg, params));
         } else if (arg == "-m" || arg == "--model") {
             params.model = get_next_arg(i, argc, argv, arg, params);
-        } else if (arg == "-i" || arg == "--input") {
-            params.input_text = get_next_arg(i, argc, argv, arg, params);
-        } else if (arg == "-l" || arg == "--tgt-lang") {
-            params.tgt_lang = get_next_arg(i, argc, argv, arg, params);
         } else if (arg == "--text") {
             params.text = true;
         } else if (arg == "-b" || arg == "--beam-size") {
@@ -87,9 +77,7 @@ bool unity_params_parse(int argc, char ** argv, unity_params & params) {
             params.opts.mem_mb = std::stoi(get_next_arg(i, argc, argv, arg, params));
         } else if (arg == "--max-audio") {
             params.max_audio_s = std::stoi(get_next_arg(i, argc, argv, arg, params));
-        } else {
-            params.files.push_back(std::string(arg));
-        }
+        } 
     }
     return true;
 }
@@ -114,87 +102,83 @@ int main(int argc, char ** argv) {
     int ctx_size_mb = params.opts.mem_mb;
     auto encoder_buf = std::vector<uint8_t>(8 * 1024 * 1024); // Only tensor metadata goes in there
     auto encoder_fwd_buf = std::vector<uint8_t>(ctx_size_mb * 1024 * 1024 / 2);
-    ggml_allocr* fwd_alloc = ggml_allocr_new(encoder_fwd_buf.data(), encoder_fwd_buf.capacity(), 8);
-    char result_str[4096];
-
-    std::string input;
-    bool interactive = (params.files.size() == 0 && params.input_text.length() == 0);
-    auto next_file = params.files.begin();
 
-    // Flag for the input case: true --> s2st, false --> t2tt
-    bool s2st_or_t2tt = true;
-
-    // S2ST
     while (true) {
-        if (interactive) {
+        // S2ST
+        if (!params.text) {
+            std::string input;
             std::cout << "\nEnter audio_path and tgt_lang, separated by space (or 'exit' to quit):\n";
             std::getline(std::cin, input);
             if (input == "exit") {
                 break;
             }
-        } else {
-            if (params.input_text.length() > 0) {
-                break;
-            }
-            if (next_file == params.files.end() && s2st_or_t2tt) break;
-            input = *(next_file++);
-        }
-        std::istringstream iss(input);
-        std::string audio_path;
-        std::string tgt_lang = params.tgt_lang;
-        iss >> audio_path >> tgt_lang;
-        if (audio_path == "-") {
-            audio_path = "/proc/self/fd/0";
-        }
-        std::cerr << "Translating (Transcribing) " << audio_path << " to " << tgt_lang << "\n";
-        SF_INFO info;
-        SNDFILE* sndfile = sf_open(audio_path.c_str(), SFM_READ, &info);
-        if (!sndfile) {
-            std::cerr << "Could not open file\n";
-            if (interactive) continue;
-            else return 1;
-        }
-        // Load audio input
-        GGML_ASSERT(info.samplerate == 16000);
-        GGML_ASSERT(info.channels == 1);
-        // Truncate audio input. Ideally we should chunk it, but this will prevent most obvious OOM.
-        int n_frames = std::min(info.samplerate * params.max_audio_s, (int)info.frames);
-        std::vector<float> data(n_frames * info.channels);
-        sf_readf_float(sndfile, data.data(), n_frames);
-
-        Result result = unity_eval_speech(model, data, params.opts, tgt_lang, params.n_threads);
-        std::string concat_transcription = std::accumulate(std::next(result.transcription.begin()), result.transcription.end(), result.transcription[0],
-            [](const std::string& a, const std::string& b) {
-                return a + " " + b;
+            std::istringstream iss(input);
+            std::string audio_path;
+            std::string tgt_lang;
+            iss >> audio_path >> tgt_lang;
+            if (audio_path == "-") {
+                audio_path = "/proc/self/fd/0";
             }
-        );
-        if (params.verbose) {
-            std::cout << "Final transcription: " << concat_transcription << std::endl;
-            std::cout << std::endl;
-            std::cout << "Word level confidence score:" << std::endl;
-            for (size_t i = 0; i < result.transcription.size(); ++i) {
-                std::cout << "Word: " << result.transcription[i] << " | Score: " << result.word_confidence_scores[i] << std::endl;
+            std::cerr << "Translating (Transcribing) " << audio_path << " to " << tgt_lang << "\n";
+            SF_INFO info;
+            SNDFILE* sndfile = sf_open(audio_path.c_str(), SFM_READ, &info);
+            if (!sndfile) {
+                std::cerr << "Could not open file\n";
+                continue;
             }
-            std::cout << std::endl;
-            std::cout << "LID scores: " << std::endl;
-            for (const auto& kv : result.lid_scores) {
-                std::cout << "Language: " << kv.first << "| Score: " << kv.second << std::endl;
+            // Load audio input
+            GGML_ASSERT(info.samplerate == 16000);
+            GGML_ASSERT(info.channels == 1);
+            // Truncate audio input. Ideally we should chunk it, but this will prevent most obvious OOM.
+            int n_frames = std::min(info.samplerate * params.max_audio_s, (int)info.frames);
+            std::vector<float> data(n_frames * info.channels);
+            sf_readf_float(sndfile, data.data(), n_frames);
+            Result result = unity_eval_speech(model, data, params.opts, tgt_lang, params.n_threads);
+            std::string concat_transcription = std::accumulate(std::next(result.transcription.begin()), result.transcription.end(), result.transcription[0],
+                [](const std::string& a, const std::string& b) {
+                    return a + " " + b;
+                }
+            );
+            if (params.verbose) {
+                std::cout << "Final transcription: " << concat_transcription << std::endl;
+                std::cout << std::endl;
+                std::cout << "Word level confidence score:" << std::endl;
+                for (size_t i = 0; i < result.transcription.size(); ++i) {
+                    std::cout << "Word: " << result.transcription[i] << " | Score: " << result.word_confidence_scores[i] << std::endl;
+                }
+                std::cout << std::endl;
+                std::cout << "LID scores: " << std::endl;
+                for (const auto& kv : result.lid_scores) {
+                    std::cout << "Language: " << kv.first << "| Score: " << kv.second << std::endl;
+                }
+            } else {
+                std::cout << concat_transcription << std::endl;
             }
+        // T2TT
         } else {
-            std::cout << concat_transcription << std::endl;
-        }
-    }
-
-    // T2TT
-    if (params.input_text.length() > 0) {
-        // tokenize the input text
-        Result result = unity_eval_text(model, params.input_text, params.opts, params.tgt_lang, params.n_threads);
-        std::string concat_translation = std::accumulate(std::next(result.transcription.begin()), result.transcription.end(), result.transcription[0],
-            [](const std::string& a, const std::string& b) {
-                return a + " " + b;
+            std::string line;
+            std::string input_text;
+            std::string tgt_lang;
+            std::cout << "\nEnter input_text and tgt_lang, separated by space (or 'exit' to quit):\n";
+            if (std::getline(std::cin, line)) {
+                std::size_t last_space = line.find_last_of(' ');
+                if (last_space != std::string::npos) {
+                    input_text = line.substr(0, last_space);
+                    tgt_lang = line.substr(last_space + 1);
+                    std::cerr << "Translating \"" << input_text << "\" to " << tgt_lang << "\n";
+                } else {
+                    std::cout << "No spaces found in the input. \n";
+                }
             }
-        );
-        std::cout << "Translation: " << concat_translation << std::endl;
+            // tokenize the input text
+            Result result = unity_eval_text(model, input_text, params.opts, tgt_lang, params.n_threads);
+            std::string concat_translation = std::accumulate(std::next(result.transcription.begin()), result.transcription.end(), result.transcription[0],
+                [](const std::string& a, const std::string& b) {
+                    return a + " " + b;
+                }
+            );
+            std::cout << "Translation: " << concat_translation << std::endl;
+        }
     }
 
     return 0;