seamless_communication/tests/integration/models/test_unity2_aligner.py

# Copyright (c) Meta Platforms, Inc. and affiliates
# All rights reserved.
#
# This source code is licensed under the license found in the
# LICENSE file in the root directory of this source tree.

from typing import Final

import torch
from fairseq2.typing import Device
from torch import tensor

from tests.common import assert_equal, device
from seamless_communication.models.aligner.alignment_extractor import AlignmentExtractor
from fairseq2.data.audio import (
    AudioDecoder,
    AudioDecoderOutput
)
from fairseq2.memory import MemoryBlock
from urllib.request import urlretrieve
import tempfile
from tests.common import assert_equal, device

REF_TEXT = "the examination and testimony of the experts enabled the commision to conclude that five shots may have been fired"

REF_DURATIONS: Final = [[ 1,  1,  2,  1,  1,  5,  5,  6,  4,  3,  2,  3,  4,  4,  2,  2,  2,  1,
           1,  1,  3,  3,  3,  4,  3,  3,  4,  3,  4,  3,  2,  2,  1,  1,  1,  1,
           2,  4,  6,  5,  4,  3,  4,  5,  5, 16,  6,  3,  5,  5,  3,  3,  1,  2,
           1,  1,  1,  2,  3,  2,  3,  1,  3,  3,  3,  2,  2,  4,  2,  2,  2,  3,
           2,  4,  5,  4,  5,  8,  3, 17,  2,  2,  3,  2,  5,  4,  6,  3,  1,  1,
           4,  4,  3,  5,  3,  3,  2,  2,  2,  2,  2,  2,  2,  1,  2,  2,  1,  1,
           2,  6,  4,  5,  9,  5,  1, 12]]

def test_aligner(example_rate16k_audio: AudioDecoderOutput) -> None:

    aligner_name = "nar_t2u_aligner"
    unit_extractor_name = "xlsr2_1b_v2"
    unit_extractor_output_layer_n = 35
    unit_extractor_kmeans_uri = "https://dl.fbaipublicfiles.com/seamlessM4T/models/unit_extraction/kmeans_10k.npy"

    extractor = AlignmentExtractor(
        aligner_name,
        unit_extractor_name,
        unit_extractor_output_layer_n,
        unit_extractor_kmeans_uri,
        device=device
    )

    audio = example_rate16k_audio["waveform"].mean(1)  # averaging mono to get [Time] shape required by aligner

    alignment_durations, _, _ = extractor.extract_alignment(audio, REF_TEXT, plot=False, add_trailing_silence=True)

    assert_equal(alignment_durations, tensor(REF_DURATIONS, device=device, dtype=torch.int64))