@inproceedings{BaumannWagnerSchusteretal.2024, author = {Baumann, Ilja and Wagner, Dominik and Schuster, Maria and Riedhammer, Korbinian and N{\"o}th, Elmar and Bocklet, Tobias}, title = {Towards Self-Attention Understanding for Automatic Articulatory Processes Analysis in Cleft Lip and Palate Speech}, doi = {10.21437/Interspeech.2024-2134}, pages = {2430 -- 2434}, year = {2024}, abstract = {Cleft lip and palate (CLP) speech presents unique challenges for automatic phoneme analysis due to its distinct acoustic characteristics and articulatory anomalies. We perform phoneme analysis in CLP speech using a pre-trained wav2vec 2.0 model with a multi-head self-attention classification module to capture long-range dependencies within the speech signal, thereby enabling better contextual understanding of phoneme sequences. We demonstrate the effectiveness of our approach in the classification of various articulatory processes in CLP speech. Furthermore, we investigate the interpretability of self-attention to gain insights into the model's understanding of CLP speech characteristics. Our findings highlight the potential of the selfattention mechanisms for improving automatic phoneme analysis in CLP speech, paving the way for enhanced diagnostics, adding interpretability for therapists and affected patients.}, language = {en} } @inproceedings{WagnerBaumannRiedhammeretal.2024, author = {Wagner, Dominik and Baumann, Ilja and Riedhammer, Korbinian and Bocklet, Tobias}, title = {Outlier Reduction with Gated Attention for Improved Post-training Quantization in Large Sequence-to-sequence Speech Foundation Models}, editor = {Pesak, Krisztina}, doi = {10.21437/Interspeech.2024-2105}, pages = {4623 -- 4627}, year = {2024}, abstract = {This paper explores the improvement of post-training quantization (PTQ) after knowledge distillation in the Whisper speech foundation model family. We address the challenge of outliers in weights and activation tensors, known to impede quantization quality in transformer-based language and vision models. Extending this observation to Whisper, we demonstrate that these outliers are also present when transformer-based models are trained to perform automatic speech recognition, necessitating mitigation strategies for PTQ. We show that outliers can be reduced by a recently proposed gating mechanism in the attention blocks of the student model, enabling effective 8-bit quantization, and lower word error rates compared to student models without the gating mechanism in place.}, language = {en} } @inproceedings{RanzenbergerFreierReinoldetal.2024, author = {Ranzenberger, Thomas and Freier, Carolin and Reinold, Luca and Riedhammer, Korbinian and Schneider, Fabian and Simic, Christopher and Simon, Claudia and Freisinger, Steffen and Georges, Munir and Bocklet, Tobias}, title = {A Multidisciplinary Approach to AI-based self-motivated Learning and Teaching with Large Language Models}, series = {Proceedings of DELFI 2024}, booktitle = {Proceedings of DELFI 2024}, publisher = {Gesellschaft f{\"u}r Informatik e.V.}, address = {Bonn}, issn = {2944-7682}, doi = {10.18420/delfi2024_11}, pages = {133-140}, year = {2024}, abstract = {We present a learning experience platform that uses machine learning methods to support students and lecturers in self-motivated online learning and teaching processes. The platform is being developed as an agile open-source collaborative project supported by multiple universities and partners. The development is guided didactically, reviewed, and scientifically evaluated in several cycles. Transparency, data protection and the copyright compliant use of the system is a central part of the project. The system further employs large language models (LLMs). Due to privacy concerns, we utilize locally hosted LLM instances and explicitly do not rely on available cloud products. Students and lecturers can interact with an LLM-based chatbot in the current prototype. The AI-generated outputs contain cross-references to the current educational video's context, indicating if sections are based on the lectures context or world knowledge. We present the prototype and results of our qualitative evaluation from the perspective of lecturers and students.}, language = {en} } @article{EscobarGrisalesRiosUrregoBaumannetal.2024, author = {Escobar-Grisales, Daniel and R{\´i}os-Urrego, Cristian-David and Baumann, Ilja and Riedhammer, Korbinian and N{\"o}th, Elmar and Bocklet, Tobias and Garcia, Adolfo and Orozco-Arroyave, Juan rafael}, title = {It's Time to Take Action: Acoustic Modeling of Motor Verbs to Detect Parkinson's Disease}, doi = {10.21437/Interspeech.2024-2205}, year = {2024}, abstract = {Pre-trained models generate speech representations that are used in different tasks, including the automatic detection of Parkinson's disease (PD). Although these models can yield high accuracy, their interpretation is still challenging. This paper used a pre-trained Wav2vec 2.0 model to represent speech frames of 25ms length and perform a frame-by-frame discrimination between PD patients and healthy control (HC) subjects. This fine granularity prediction enabled us to identify specific linguistic segments with high discrimination capability. Speech representations of all produced verbs were compared w.r.t. nouns and the first ones yielded higher accuracies. To gaina deeper understanding of this pattern, representations of motor and non-motor verbs were compared and the first ones yielded better results, with accuracies of around 83\% in an independent test set. These findings support well-established neurocognitive models about action-related language highlighted as key drivers of PD. Index Terms: computational paralinguistics, interpretability of pre-trained models, action verbs, Parkinson's disease}, language = {en} } @inproceedings{BraunWitzlErzigkeitetal.2025, author = {Braun, Franziska and Witzl, Christopher and Erzigkeit, Andreas and Lehfeld, Hartmut and Hillemacher, Thomas and Bocklet, Tobias and Riedhammer, Korbinian}, title = {Pitfalls and Limits in Automatic Dementia Assessment}, series = {Interspeech 2025}, booktitle = {Interspeech 2025}, publisher = {ISCA}, issn = {2958-1796}, doi = {10.21437/Interspeech.2025-2280}, pages = {5663 -- 5667}, year = {2025}, abstract = {Current work on speech-based dementia assessment focuses on either feature extraction to predict assessment scales, or on the automation of existing test procedures. Most research uses public data unquestioningly and rarely performs a detailed error analysis, focusing primarily on numerical performance. We perform an in-depth analysis of an automated standardized dementia assessment, the Syndrom-Kurz-Test. We find that while there is a high overall correlation with human annotators, due to certain artifacts, we observe high correlations for the severely impaired individuals, which is less true for the healthy or mildly impaired ones. Speech production decreases with cognitive decline, leading to overoptimistic correlations when test scoring relies on word naming. Depending on the test design, fallback handling introduces further biases that favor certain groups. These pitfalls remain independent of group distributions in datasets and require differentiated analysis of target groups.}, language = {en} } @inproceedings{BundschererSchmittBaumannetal.2025, author = {Bundscherer, Maximilian and Schmitt, Thomas and Baumann, Ilja and Bocklet, Tobias}, title = {Digital Operating Mode Classification of Real-World Amateur Radio Transmissions}, series = {ICASSP 2025 - 2025 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, booktitle = {ICASSP 2025 - 2025 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, publisher = {IEEE}, doi = {10.1109/ICASSP49660.2025.10889837}, pages = {1 -- 5}, year = {2025}, abstract = {This study presents an ML approach for classifying digital radio operating modes evaluated on real-world transmissions. We generated 98 different parameterized radio signals from 17 digital operating modes, transmitted each of them on the 70 cm (UHF) amateur radio band, and recorded our transmissions with two different architectures of SDR receivers. Three lightweight ML models were trained exclusively on spectrograms of limited non-transmitted signals with random characters as payloads. This training involved an online data augmentation pipeline to simulate various radio channel impairments. Our best model, EfficientNetB0, achieved an accuracy of 93.80\% across the 17 operating modes and 85.47\% across all 98 parameterized radio signals, evaluated on our real-world transmissions with Wikipedia articles as payloads. Furthermore, we analyzed the impact of varying signal durations \& the number of FFT bins on classification, assessed the effectiveness of our simulated channel impairments, and tested our models across multiple simulated SNRs.}, language = {en} } @inproceedings{BaumannWagnerRiedhammeretal.2025, author = {Baumann, Ilja and Wagner, Dominik and Riedhammer, Korbinian and Bocklet, Tobias}, title = {Optimized Self-supervised Training with BEST-RQ for Speech Recognition}, series = {ICASSP 2025 - 2025 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, booktitle = {ICASSP 2025 - 2025 IEEE International Conference on Acoustics, Speech and Signal Processing (ICASSP)}, publisher = {IEEE}, doi = {10.1109/ICASSP49660.2025.10889362}, pages = {1 -- 5}, year = {2025}, abstract = {Self-supervised learning has been successfully used for various speech related tasks, including automatic speech recognition. BERT-based Speech pre-Training with Random-projection Quantizer (BEST-RQ) has achieved state-of-the-art results in speech recognition. In this work, we further optimize the BEST-RQ approach using Kullback-Leibler divergence as an additional regularizing loss and multicodebook extension per cluster derived from low-level feature clustering. Preliminary experiments on train-100 split of LibriSpeech result in a relative improvement of 11.2\% on test-clean by using multiple codebooks, utilizing a combination of cross-entropy and Kullback-Leibler divergence further reduces the word error rate by 4.5\%. The proposed optimizations on full LibriSpeech pre-training and fine-tuning result in relative word error rate improvements of up to 23.8\% on test-clean and 30.6\% on testother using 6 codebooks. Furthermore, the proposed setup leads to faster convergence in pre-training and fine-tuning and additionally stabilizes the pre-training.}, language = {en} } @inproceedings{FreisingerSeebergerRanzenbergeretal.2025, author = {Freisinger, Steffen and Seeberger, Philipp and Ranzenberger, Thomas and Bocklet, Tobias and Riedhammer, Korbinian}, title = {Towards Multi-Level Transcript Segmentation: LoRA Fine-Tuning for Table-of-Contents Generation}, series = {Interspeech 2025}, booktitle = {Interspeech 2025}, publisher = {ISCA}, address = {ISCA}, issn = {2958-1796}, doi = {10.21437/Interspeech.2025-2792}, pages = {276 -- 280}, year = {2025}, abstract = {Segmenting speech transcripts into thematic sections benefits both downstream processing and users who depend on written text for accessibility. We introduce a novel approach to hierarchical topic segmentation in transcripts, generating multi-level tables of contents that capture both topic and subtopic boundaries. We compare zero-shot prompting and LoRA fine-tuning on large language models, while also exploring the integration of high-level speech pause features. Evaluations on English meeting recordings and multilingual lecture transcripts (Portuguese, German) show significant improvements over established topic segmentation baselines. Additionally, we adapt a common evaluation measure for multi-level segmentation, taking into account all hierarchical levels within one metric.}, language = {en} } @article{LeeBokLeeWagneretal.2025, author = {Lee, Seanie and Bok Lee, Dong and Wagner, Dominik and Kang, Minki and Seong, Haebin and Bocklet, Tobias and Lee, Juho and Hwang, Sung Ju}, title = {SafeRoute: Adaptive Model Selection for Efficient and Accurate Safety Guardrails in Large Language Models,}, series = {Findings of the Association for Computational Linguistics: ACL 2025}, journal = {Findings of the Association for Computational Linguistics: ACL 2025}, publisher = {Association for Computational Linguistics}, address = {Vienna, Austria}, pages = {2053-2069}, year = {2025}, abstract = {Deploying large language models (LLMs) in real-world applications requires robust safety guard models to detect and block harmful user prompts. While large safety guard models achieve strong performance, their computational cost is substantial. To mitigate this, smaller distilled models are used, but they often underperform on "hard" examples where the larger model provides accurate predictions. We observe that many inputs can be reliablyhandled by the smaller model, while only a small fraction require the larger model's capacity. Motivated by this, we propose SafeRoute, a binary router that distinguishes hard examples from easy ones. Our method selectively applies the larger safety guard model to the data that the router considers hard, improving efficiency while maintaining accuracy compared to solely using the larger safety guard model. Experimental results on multiple benchmark datasets demonstrate that our adaptive model selection significantly enhances the trade-off between computational cost and safety performance, outperforming relevant baselines.}, language = {en} } @inproceedings{WagnerBaumannEngertetal.2025, author = {Wagner, Dominik and Baumann, Ilja and Engert, Natalie and Lee, Seanie and N{\"o}th, Elmar and Riedhammer, Korbinian and Bocklet, Tobias}, title = {Personalized Fine-Tuning with Controllable Synthetic Speech from LLM-Generated Transcripts for Dysarthric Speech Recognition}, series = {Interspeech 2025}, booktitle = {Interspeech 2025}, publisher = {ISCA}, address = {ISCA}, issn = {2958-1796}, doi = {10.21437/Interspeech.2025-2155}, pages = {3294 -- 3298}, year = {2025}, abstract = {In this work, we present our submission to the Speech Accessibility Project challenge for dysarthric speech recognition. We integrate parameter-efficient fine-tuning with latent audio representations to improve an encoder-decoder ASR system. Synthetic training data is generated by fine-tuning Parler-TTS to mimic dysarthric speech, using LLM-generated prompts for corpus-consistent target transcripts. Personalization with x-vectors consistently reduces word error rates (WERs) over non-personalized fine-tuning. AdaLoRA adapters outperform full fine-tuning and standard low-rank adaptation, achieving relative WER reductions of ∼23\% and ∼22\%, respectively. Further improvements (∼5\% WER reduction) come from incorporating wav2vec 2.0-based audio representations. Training with synthetic dysarthric speech yields up to ∼7\% relative WER improvement over personalized fine-tuning alone.}, language = {en} } @inproceedings{BaumannWagnerRiedhammeretal.2025, author = {Baumann, Ilja and Wagner, Dominik and Riedhammer, Korbinian and Bocklet, Tobias}, title = {Pathology-Aware Speech Encoding and Data Augmentation for Dysarthric Speech Recognition}, series = {Interspeech 2025}, booktitle = {Interspeech 2025}, publisher = {ISCA}, address = {ISCA}, issn = {2958-1796}, doi = {10.21437/Interspeech.2025-2724}, pages = {3289 -- 3293}, year = {2025}, abstract = {Automatic speech recognition (ASR) for pathologic speech remains a major challenge due to high variability in articulation, phonation, and prosody distortions. In this work, we propose a pathology-aware speech encoder based on BEST-RQ pre-training, which incorporates 46k hours of speech, including pathologic and atypical speech. We continue pre-training for domain adaptation and experiment with etiology-specific codebooks. We achieve a 13.2\% relative word error rate (WER) improvement using the pathology-aware speech encoder with etiology-specific continued pre-training. Additionally, we examine the impact of incorporating synthetic and out-of-domain (OOD) data to further enhance ASR performance. Synthetic data reduces WER by up to 8.7\%, while OOD data improves WER by 12.2\%. Finally, we introduce a semantic similaritybased data augmentation technique to optimize data selection, achieving a WER improvement of up to 9.7\% while minimizing the need for additional training data.}, language = {en} } @inproceedings{WagnerBaumannBocklet2025, author = {Wagner, Dominik and Baumann, Ilja and Bocklet, Tobias}, title = {Vocoder-Free Non-parallel Conversion of Whispered Speech With Masked Cycle-Consistent Generative Adversarial Networks}, publisher = {Springer}, address = {Cham}, isbn = {978-3-032-02548-7}, doi = {10.1007/978-3-032-02548-7_20}, pages = {235-246}, year = {2025}, abstract = {Cycle-consistent generative adversarial networks have been widely used in non-parallel voice conversion (VC). Their ability to learn mappings between source and target features without relying on parallel training data eliminates the need for temporal alignments. However, most methods decouple the conversion of acoustic features from synthesizing the audio signal by using separate models for conversion and waveform synthesis. This work unifies conversion and synthesis into a single model, thereby eliminating the need for a separate vocoder. By leveraging cycle-consistent training and a self-supervised auxiliary training task, our model is able to efficiently generate converted high-quality raw audio waveforms. Subjective listening tests showed that our unified approach achieved improvements of up to 6.7\% relative to the baseline in whispered VC. Mean opinion score predictions also yielded stable results in conventional VC (between 0.5\% and 2.4\% relative improvement).}, language = {en} } @article{RanzenbergerBaumannBayerletal.2025, author = {Ranzenberger, Thomas and Baumann, Ilja and Bayerl, Sebastian and Wagner, Dominik and Bocklet, Tobias and Riedhammer, Korbinian}, title = {Evaluation of recognition errors of hybrid and transformer-based ASR systems in German video lectures}, series = {Studientexte zur Sprachkommunikation: Elektronische Sprachsignalverarbeitung 2025 - Book}, journal = {Studientexte zur Sprachkommunikation: Elektronische Sprachsignalverarbeitung 2025 - Book}, publisher = {ESSV 2025}, address = {Halle, Deutschland}, pages = {101-108}, year = {2025}, abstract = {We analyze different errors in speech recognition systems, focusing on consecutive insertions and deletions, known as hallucinations and elisions in transformer-based end-to-end automatic speech recognition (ASR) systems. We compare errors from a TDNN-HMM, and whisper-based models on English and German spontaneous speech. Based on a human annotated subset of German lecture videos, we investigate whether these blocks of deletions affect the semantics of the utterance. Whisper performs best and preserves the meaning in 90\% of the annotated error segments even containing consecutive deletions on this subset. We analyze the word error rate and do further analysis of errors using natural language processing to detect lemmatization errors, compound word errors, and out-of-vocabulary words. We discuss possible reasons and mitigations.}, language = {en} }