SSLAM_AS2M_Finetuned

🔧 Usage

You can load and use the model for feature extraction directly via Hugging Face Transformers:

import torchaudio
import torch
import soundfile as sf
import numpy as np
from transformers import AutoModel

model_id = "ta012/SSLAM_AS2M_Finetuned"
model = AutoModel.from_pretrained(model_id, trust_remote_code=True).eval().cuda()

source_file = "/path/to/input.wav"
target_length = 1024    # Recommended: 1024 for 10s audio
norm_mean = -4.268
norm_std = 4.569

# Load and resample audio
wav, sr = sf.read(source_file)
waveform = torch.tensor(wav).float().cuda()
if sr != 16000:
    waveform = torchaudio.functional.resample(waveform, sr, 16000)

# Normalize and convert to mel-spectrogram
waveform = waveform - waveform.mean()
mel = torchaudio.compliance.kaldi.fbank(
    waveform.unsqueeze(0),
    htk_compat=True,
    sample_frequency=16000,
    use_energy=False,
    window_type='hanning',
    num_mel_bins=128,
    dither=0.0,
    frame_shift=10
).unsqueeze(0)

# Pad or truncate
n_frames = mel.shape[1]
if n_frames < target_length:
    mel = torch.nn.ZeroPad2d((0, 0, 0, target_length - n_frames))(mel)
else:
    mel = mel[:, :target_length, :]

# Normalize
mel = (mel - norm_mean) / (norm_std * 2)
mel = mel.unsqueeze(0).cuda()  # shape: [1, 1, T, F]

# Extract features
with torch.no_grad():
    feat = model.extract_features(mel)

feat = feat.squeeze(0).cpu().numpy()
print(f"Feature shape: {feat.shape}")
# end of feature extraction

## get top 12 predictions
with torch.no_grad():
    pred = model(mel)

vocab = {0: 'Speech', 1: 'Male speech, man speaking', 2: 'Female speech, woman speaking', 3: 'Child speech, kid speaking', 4: 'Conversation', 5: 'Narration, monologue', 6: 'Babbling', 7: 'Speech synthesizer', 8: 'Shout', 9: 'Bellow', 10: 'Whoop', 11: 'Yell', 12: 'Battle cry', 13: 'Children shouting', 14: 'Screaming', 15: 'Whispering', 16: 'Laughter', 17: 'Baby laughter', 18: 'Giggle', 19: 'Snicker', 20: 'Belly laugh', 21: 'Chuckle, chortle', 22: 'Crying, sobbing', 23: 'Baby cry, infant cry', 24: 'Whimper', 25: 'Wail, moan', 26: 'Sigh', 27: 'Singing', 28: 'Choir', 29: 'Yodeling', 30: 'Chant', 31: 'Mantra', 32: 'Male singing', 33: 'Female singing', 34: 'Child singing', 35: 'Synthetic singing', 36: 'Rapping', 37: 'Humming', 38: 'Groan', 39: 'Grunt', 40: 'Whistling', 41: 'Breathing', 42: 'Wheeze', 43: 'Snoring', 44: 'Gasp', 45: 'Pant', 46: 'Snort', 47: 'Cough', 48: 'Throat clearing', 49: 'Sneeze', 50: 'Sniff', 51: 'Run', 52: 'Shuffle', 53: 'Walk, footsteps', 54: 'Chewing, mastication', 55: 'Biting', 56: 'Gargling', 57: 'Stomach rumble', 58: 'Burping, eructation', 59: 'Hiccup', 60: 'Fart', 61: 'Hands', 62: 'Finger snapping', 63: 'Clapping', 64: 'Heart sounds, heartbeat', 65: 'Heart murmur', 66: 'Cheering', 67: 'Applause', 68: 'Chatter', 69: 'Crowd', 70: 'Hubbub, speech noise, speech babble', 71: 'Children playing', 72: 'Animal', 73: 'Domestic animals, pets', 74: 'Dog', 75: 'Bark', 76: 'Yip', 77: 'Howl', 78: 'Bow-wow', 79: 'Growling', 80: 'Whimper (dog)', 81: 'Cat', 82: 'Purr', 83: 'Meow', 84: 'Hiss', 85: 'Caterwaul', 86: 'Livestock, farm animals, working animals', 87: 'Horse', 88: 'Clip-clop', 89: 'Neigh, whinny', 90: 'Cattle, bovinae', 91: 'Moo', 92: 'Cowbell', 93: 'Pig', 94: 'Oink', 95: 'Goat', 96: 'Bleat', 97: 'Sheep', 98: 'Fowl', 99: 'Chicken, rooster', 100: 'Cluck', 101: 'Crowing, cock-a-doodle-doo', 102: 'Turkey', 103: 'Gobble', 104: 'Duck', 105: 'Quack', 106: 'Goose', 107: 'Honk', 108: 'Wild animals', 109: 'Roaring cats (lions, tigers)', 110: 'Roar', 111: 'Bird', 112: 'Bird vocalization, bird call, bird song', 113: 'Chirp, tweet', 114: 'Squawk', 115: 'Pigeon, dove', 116: 'Coo', 117: 'Crow', 118: 'Caw', 119: 'Owl', 120: 'Hoot', 121: 'Bird flight, flapping wings', 122: 'Canidae, dogs, wolves', 123: 'Rodents, rats, mice', 124: 'Mouse', 125: 'Patter', 126: 'Insect', 127: 'Cricket', 128: 'Mosquito', 129: 'Fly, housefly', 130: 'Buzz', 131: 'Bee, wasp, etc.', 132: 'Frog', 133: 'Croak', 134: 'Snake', 135: 'Rattle', 136: 'Whale vocalization', 137: 'Music', 138: 'Musical instrument', 139: 'Plucked string instrument', 140: 'Guitar', 141: 'Electric guitar', 142: 'Bass guitar', 143: 'Acoustic guitar', 144: 'Steel guitar, slide guitar', 145: 'Tapping (guitar technique)', 146: 'Strum', 147: 'Banjo', 148: 'Sitar', 149: 'Mandolin', 150: 'Zither', 151: 'Ukulele', 152: 'Keyboard (musical)', 153: 'Piano', 154: 'Electric piano', 155: 'Organ', 156: 'Electronic organ', 157: 'Hammond organ', 158: 'Synthesizer', 159: 'Sampler', 160: 'Harpsichord', 161: 'Percussion', 162: 'Drum kit', 163: 'Drum machine', 164: 'Drum', 165: 'Snare drum', 166: 'Rimshot', 167: 'Drum roll', 168: 'Bass drum', 169: 'Timpani', 170: 'Tabla', 171: 'Cymbal', 172: 'Hi-hat', 173: 'Wood block', 174: 'Tambourine', 175: 'Rattle (instrument)', 176: 'Maraca', 177: 'Gong', 178: 'Tubular bells', 179: 'Mallet percussion', 180: 'Marimba, xylophone', 181: 'Glockenspiel', 182: 'Vibraphone', 183: 'Steelpan', 184: 'Orchestra', 185: 'Brass instrument', 186: 'French horn', 187: 'Trumpet', 188: 'Trombone', 189: 'Bowed string instrument', 190: 'String section', 191: 'Violin, fiddle', 192: 'Pizzicato', 193: 'Cello', 194: 'Double bass', 195: 'Wind instrument, woodwind instrument', 196: 'Flute', 197: 'Saxophone', 198: 'Clarinet', 199: 'Harp', 200: 'Bell', 201: 'Church bell', 202: 'Jingle bell', 203: 'Bicycle bell', 204: 'Tuning fork', 205: 'Chime', 206: 'Wind chime', 207: 'Change ringing (campanology)', 208: 'Harmonica', 209: 'Accordion', 210: 'Bagpipes', 211: 'Didgeridoo', 212: 'Shofar', 213: 'Theremin', 214: 'Singing bowl', 215: 'Scratching (performance technique)', 216: 'Pop music', 217: 'Hip hop music', 218: 'Beatboxing', 219: 'Rock music', 2