@inproceedings{SimicBocklet2024, author = {Simic, Christopher and Bocklet, Tobias}, title = {Self-Supervised Adaptive AV Fusion Module for Pre-Trained ASR Models}, series = {ICASSP 2024 - 2024 IEEE International}, booktitle = {ICASSP 2024 - 2024 IEEE International}, doi = {10.1109/ICASSP48485.2024.10448047}, pages = {12787 -- 12791}, year = {2024}, language = {en} } @inproceedings{BaumannWagnerSchusteretal.2024, author = {Baumann, Ilja and Wagner, Dominik and Schuster, Maria and N{\"o}th, Elmar and Bocklet, Tobias}, title = {Towards Interpretability of Automatic Phoneme Analysis in Cleft Lip and Palate Speech}, series = {IEEE International Conference on Acoustics, Speech, and Signal Processing (ICASSP)}, booktitle = {IEEE International Conference on Acoustics, Speech, and Signal Processing (ICASSP)}, doi = {10.1109/ICASSP48485.2024.10447632}, pages = {12602 -- 12606}, year = {2024}, abstract = {Cleft Lip and Palate ranks among the most common congenital abnormalities and significantly influences speech articulation, resulting in varying phonemic impacts. In a clinical context, a detailed diagnosis is carried out by time-consuming perceptual evaluations. We use perceptual ratings of different articulatory modifications on phoneme-level as ground-truth and propose a system based on wav2vec 2.0, trained to the downstream task of classifying phonemic criteria as a multi-class and multi-label problem. The system is trained for detection on utterance level, without the usage of phoneme labels. To gain a clearer understanding of which areas of the speech signal have the greatest impact on classification, we assess the extent to which our system aligns with expert ratings at the phoneme level. Additionally, we examine which specific phonemes play a decisive role in determining the final classification of the labeled criteria. The results show that salient phonemes marked by experts contribute remarkably greater to the classification of the correct class using feature relevance explanation methods. To the best of our knowledge, this is the first study incorporating various utterance-level articulatory modifications classification and phoneme-level interpretation, offering a more comprehensive understanding for potential clinical applications.}, language = {en} } @inproceedings{RanzenbergerFreierReinoldetal.2024, author = {Ranzenberger, Thomas and Freier, Carolin and Reinold, Luca and Riedhammer, Korbinian and Schneider, Fabian and Simic, Christopher and Simon, Claudia and Freisinger, Steffen and Georges, Munir and Bocklet, Tobias}, title = {A Multidisciplinary Approach to AI-based self-motivated Learning and Teaching with Large Language Models}, series = {Proceedings of DELFI Workshops 2024}, booktitle = {Proceedings of DELFI Workshops 2024}, publisher = {Gesellschaft f{\"u}r Informatik e.V.}, doi = {10.18420/delfi2024_11}, year = {2024}, abstract = {We present a learning experience platform that uses machine learning methods to support students and lecturers in self-motivated online learning and teaching processes. The platform is being developed as an agile open-source collaborative project supported by multiple universities and partners. The development is guided didactically, reviewed, and scientifically evaluated in several cycles. Transparency, data protection and the copyright compliant use of the system is a central part of the project. The system further employs large language models (LLMs). Due to privacy concerns, we utilize locally hosted LLM instances and explicitly do not rely on available cloud products. Students and lecturers can interact with an LLM-based chatbot in the current prototype. The AI-generated outputs contain cross-references to the current educational video's context, indicating if sections are based on the lectures context or world knowledge. We present the prototype and results of our qualitative evaluation from the perspective of lecturers and students.}, language = {en} } @inproceedings{BraunBayerlHoenigetal.2024, author = {Braun, Franziska and Bayerl, Sebastian and H{\"o}nig, Florian and Lehfeld, Hartmut and Hillemacher, Thomas and Bocklet, Tobias and Riedhammer, Korbinian}, title = {Infusing Acoustic Pause Context into Text-Based Dementia Assessment}, issn = {2958-1796}, doi = {10.21437/Interspeech.2024-2496}, pages = {1980-1984}, year = {2024}, abstract = {Speech pauses, alongside content and structure, offer a valuable and non-invasive biomarker for detecting dementia. This work investigates the use of pause-enriched transcripts in transformer-based language models to differentiate the cognitive states of subjects with no cognitive impairment, mild cognitive impairment, and Alzheimer's dementia based on their speech from a clinical assessment. We address three binary classification tasks: Onset, monitoring, and dementia exclusion. The performance is evaluated through experiments on a German Verbal Fluency Test and a Picture Description Test, comparing the model's effectiveness across different speech production contexts. Starting from a textual baseline, we investigate the effect of incorporation of pause information and acoustic context. We show the test should be chosen depending on the task, and similarly, lexical pause information and acoustic cross-attention contribute differently.}, language = {en} } @inproceedings{WagnerBaumannRanzenbergeretal.2024, author = {Wagner, Dominik and Baumann, Ilja and Ranzenberger, Thomas and Riedhammer, Korbinian and Bocklet, Tobias}, title = {Personalizing Large Sequence-to-Sequence Speech Foundation Models With Speaker Representations}, doi = {10.1109/SLT61566.2024.10832252}, pages = {1-6}, year = {2024}, abstract = {We present a method to personalize large transformer-based encoderdecoder speech foundation models without the need for changes in the underlying model structure or training from scratch. This is achieved by projecting speaker-specific information into the latent space of the transformer decoder via a small neural network and learning to process the speaker information along with domainspecific information via parameter-efficient finetuning. We use this method to improve the automatic speech recognition results of spoken academic German and English. Our approach yields average relative word error rate (WER) improvements of approximately 29\% on German academic speech and 25\% on English academic speech. It also translates well to conversational speech, achieving relative WER improvements of up to 36\%, and demonstrates modest gains of up to 5\% on read speech. Moreover, we observe that incorporating utterances from the recent past as personalization context yields the most significant overall improvements and that changes in voice characteristics resulting from prolonged speaking have a minimal effect on the personalization quality of academic lectures.}, language = {en} } @article{WagnerBaumannBocklet2024, author = {Wagner, Dominik and Baumann, Ilja and Bocklet, Tobias}, title = {Generative adversarial networks for whispered to voiced speech conversion: a comparative study}, series = {International Journal of Speech Technology}, volume = {27}, journal = {International Journal of Speech Technology}, editor = {Baumann, Ilja}, doi = {10.1007/s10772-024-10161-1}, pages = {1093-1110}, year = {2024}, abstract = {Generative Adversarial Networks (GANs) have demonstrated promising results as end-to-end models for whispered to voiced speech conversion. Leveraging non-autoregressive systems like GANs capable of performing conditional waveform generation eliminates the need for separate models to estimate voiced speech features, and leads to faster inference compared to autoregressive methods. This study aims to identify the optimal GAN architecture for the whispered to voiced speech conversion task by comparing six state-of-the-art models. Furthermore, we present a method for evaluating the preservation of speaker identity and local accent, using embeddings obtained from speaker- and language identification systems. Our experimental results show that building the speech conversion system based on the HiFi-GAN architecture yields the best objective evaluation scores, outperforming the baseline by ∼9\% relative using frequency-weighted Signal-to-Noise Ratio and Log Likelihood Ratio, as well as by ∼29\% relative using Root Mean Squared Error. In subjective tests, HiFi-GAN yielded a mean opinion score of 2.9, significantly outperforming the baseline with a score of 1.4. Furthermore, HiFi-GAN enhanced ASR performance and preserved speaker identity and accent, with correct language detection rates of up to ∼98\%.}, language = {en} } @inproceedings{WagnerLeeBaumannetal.2024, author = {Wagner, Dominik and Lee, Seanie and Baumann, Ilja and Seeberger, Philipp and Riedhammer, Korbinian and Bocklet, Tobias}, title = {Optimized Speculative Sampling for {GPU} Hardware Accelerators}, series = {Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing}, booktitle = {Proceedings of the 2024 Conference on Empirical Methods in Natural Language Processing}, publisher = {Association for Computational Linguistics}, address = {Miami, Florida, USA}, doi = {10.18653/v1/2024.emnlp-main.370}, pages = {6442-6458}, year = {2024}, abstract = {In this work, we optimize speculative sampling for parallel hardware accelerators to improve sampling speed. We notice that substantial portions of the intermediate matrices necessary for speculative sampling can be computed concurrently. This allows us to distribute the workload across multiple GPU threads, enabling simultaneous operations on matrix segments within thread blocks. This results in profiling time improvements ranging from 6\% to 13\% relative to the baseline implementation, without compromising accuracy. To further accelerate speculative sampling, probability distributions parameterized by softmax are approximated by sigmoid. This approximation approach results in significantly greater relative improvements in profiling time, ranging from 37\% to 94\%, with a minor decline in accuracy. We conduct extensive experiments on both automatic speech recognition and summarization tasks to validate the effectiveness of our optimization methods.}, language = {en} } @inproceedings{RanzenbergerBockletFreisingeretal.2024, author = {Ranzenberger, Thomas and Bocklet, Tobias and Freisinger, Steffen and Georges, Munir and Glockner, Kevin and Herygers, Aaricia and Riedhammer, Korbinian and Schneider, Fabian and Simic, Christopher and Zakaria, Khabbab}, title = {EXTENDING HANS: LARGE LANGUAGE MODELS FOR QUESTION ANSWERING, SUMMARIZATION, AND TOPIC SEGMENTATION IN AN ML-BASED LEARNING EXPERIENCE PLATFORM}, series = {Elektronische Sprachsignalverarbeitung 2024, Tagungsband der 35. Konferenz, Regensburg, 6.-8. M{\"a}rz 2024}, booktitle = {Elektronische Sprachsignalverarbeitung 2024, Tagungsband der 35. Konferenz, Regensburg, 6.-8. M{\"a}rz 2024}, publisher = {TUPress}, address = {Dresden}, isbn = {978-3-95908-325-6}, pages = {219-224}, year = {2024}, abstract = {Abstract: The use of chatbots based on large language models (LLMs) and their impact on society are influencing our learning experience platform Hochschul Assistenz-System (HAnS). HAnS uses machine learning (ML) methods to support students and lecturers in the online learning and teaching processes [1]. This paper introduces LLM-based features available in HAnS which are using the transcript of our improved Automatic Speech Recognition (ASR) pipeline with an average transcription duration of 45 seconds and an average word error rate (WER) of 6.66\% on over 8 hours of audio data of 7 lecture videos. A LLM-based chatbot could be used to answer questions on the lecture content as the ASR transcript is provided as context. The summarization and topic segmentation uses the LLM to improve our learning experience platform. We generate multiple choice questions using the LLM and the ASR transcript as context during playback in a period of 3 minutes and display them in the HAnS frontend}, language = {en} } @inproceedings{BundschererSchmittBocklet2024, author = {Bundscherer, Maximilian and Schmitt, Thomas and Bocklet, Tobias}, title = {Machine Learning in Industrial Quality Control of Glass Bottle Prints}, doi = {10.48550/arXiv.2409.20132}, pages = {8}, year = {2024}, abstract = {In industrial manufacturing of glass bottles, quality control of bottle prints is necessary as numerous factors can negatively affect the printing process. Even minor defects in the bottle prints must be detected despite reflections in the glass or manufacturing-related deviations. In cooperation with our medium-sized industrial partner, two ML-based approaches for quality control of these bottle prints were developed and evaluated, which can also be used in this challenging scenario. Our first approach utilized different filters to supress reflections (e.g. Sobel or Canny) and image quality metrics for image comparison (e.g. MSE or SSIM) as features for different supervised classification models (e.g. SVM or k-Neighbors), which resulted in an accuracy of 84\%. The images were aligned based on the ORB algorithm, which allowed us to estimate the rotations of the prints, which may serve as an indicator for anomalies in the manufacturing process. In our second approach, we fine-tuned different pre-trained CNN models (e.g. ResNet or VGG) for binary classification, which resulted in an accuracy of 87\%. Utilizing Grad-Cam on our fine-tuned ResNet-34, we were able to localize and visualize frequently defective bottle print regions. This method allowed us to provide insights that could be used to optimize the actual manufacturing process. This paper also describes our general approach and the challenges we encountered in practice with data collection during ongoing production, unsupervised preselection, and labeling.}, language = {en} } @inproceedings{WagnerBayerlBaumannetal.2024, author = {Wagner, Dominik and Bayerl, Sebastian P. and Baumann, Ilja and Riedhammer, Korbinian and N{\"o}th, Elmar and Bocklet, Tobias}, title = {Large Language Models for Dysfluency Detection in Stuttered Speech}, doi = {10.48550/arXiv.2406.11025}, pages = {6}, year = {2024}, abstract = {Accurately detecting dysfluencies in spoken language can help to improve the performance of automatic speech and language processing components and support the development of more inclusive speech and language technologies. Inspired by the recent trend towards the deployment of large language models (LLMs) as universal learners and processors of non-lexical inputs, such as audio and video, we approach the task of multi-label dysfluency detection as a language modeling problem. We present hypotheses candidates generated with an automatic speech recognition system and acoustic representations extracted from an audio encoder model to an LLM, and finetune the system to predict dysfluency labels on three datasets containing English and German stuttered speech. The experimental results show that our system effectively combines acoustic and lexical information and achieves competitive results on the multi-label stuttering detection task.}, language = {en} } @techreport{SchmittBundschererBocklet2024, author = {Schmitt, Thomas and Bundscherer, Maximilian and Bocklet, Tobias}, title = {Semmeldetector: Application of Machine Learning in Commercial Bakeries}, doi = {10.48550/arXiv.2406.04050}, pages = {6}, year = {2024}, abstract = {The Semmeldetector, is a machine learning application that utilizes object detection models to detect, classify and count baked goods in images. Our application allows commercial bakers to track unsold baked goods, which allows them to optimize production and increase resource efficiency. We compiled a dataset comprising 1151 images that distinguishes between 18 different types of baked goods to train our detection models. To facilitate model training, we used a Copy-Paste augmentation pipeline to expand our dataset. We trained the state-of-the-art object detection model YOLOv8 on our detection task. We tested the impact of different training data, model scale, and online image augmentation pipelines on model performance. Our overall best performing model, achieved an AP@0.5 of 89.1\% on our test set. Based on our results, we conclude that machine learning can be a valuable tool even for unforeseen industries like bakeries, even with very limited datasets.}, language = {en} } @inproceedings{BaumannUngerWagneretal.2024, author = {Baumann, Ilja and Unger, Nicole and Wagner, Dominik and Riedhammer, Korbinian and Bocklet, Tobias}, title = {Automatic Evaluation of a Sentence Memory Test for Preschool Children}, doi = {10.21437/Interspeech.2024-2125}, pages = {5158 -- 5162}, year = {2024}, abstract = {Assessment of memory capabilities in preschool-aged children is crucial for early detection of potential speech development impairments or delays. We present an approach for the automatic evaluation of a standardized sentence memory test specifically for preschool children. Our methodology leverages automatic transcription of recited sentences and evaluation based on natural language processing techniques. We demonstrate the effectiveness of our approach on a dataset comprised of recited sentences from preschool-aged children, incorporating ratings of semantic and syntactic correctness. The best performing systems achieve an F1 score of 91.7\% for semantic correctness and 86.1\% for syntactic correctness using automatic transcripts. Our results showcase the potential of automated evaluation systems in providing reliable and efficient assessments of memory capabilities in early childhood, facilitating timely interventions and support for children with language development needs.}, language = {en} } @article{SchmittBundschererBocklet2024, author = {Schmitt, Thomas and Bundscherer, Maximilian and Bocklet, Tobias}, title = {Training a Computer Vision Model for Commercial Bakeries with Primarily Synthetic Images}, doi = {10.48550/arXiv.2409.20122}, pages = {10}, year = {2024}, abstract = {In the food industry, reprocessing returned product is a vital step to increase resource efficiency. [SBB23] presented an AI application that automates the tracking of returned bread buns. We extend their work by creating an expanded dataset comprising 2432 images and a wider range of baked goods. To increase model robustness, we use generative models pix2pix and CycleGAN to create synthetic images. We train state-of-the-art object detection model YOLOv9 and YOLOv8 on our detection task. Our overall best-performing model achieved an average precision AP@0.5 of 90.3\% on our test set.}, language = {en} } @inproceedings{BaumannWagnerSchusteretal.2024, author = {Baumann, Ilja and Wagner, Dominik and Schuster, Maria and Riedhammer, Korbinian and N{\"o}th, Elmar and Bocklet, Tobias}, title = {Towards Self-Attention Understanding for Automatic Articulatory Processes Analysis in Cleft Lip and Palate Speech}, doi = {10.21437/Interspeech.2024-2134}, pages = {2430 -- 2434}, year = {2024}, abstract = {Cleft lip and palate (CLP) speech presents unique challenges for automatic phoneme analysis due to its distinct acoustic characteristics and articulatory anomalies. We perform phoneme analysis in CLP speech using a pre-trained wav2vec 2.0 model with a multi-head self-attention classification module to capture long-range dependencies within the speech signal, thereby enabling better contextual understanding of phoneme sequences. We demonstrate the effectiveness of our approach in the classification of various articulatory processes in CLP speech. Furthermore, we investigate the interpretability of self-attention to gain insights into the model's understanding of CLP speech characteristics. Our findings highlight the potential of the selfattention mechanisms for improving automatic phoneme analysis in CLP speech, paving the way for enhanced diagnostics, adding interpretability for therapists and affected patients.}, language = {en} } @inproceedings{WagnerBaumannRiedhammeretal.2024, author = {Wagner, Dominik and Baumann, Ilja and Riedhammer, Korbinian and Bocklet, Tobias}, title = {Outlier Reduction with Gated Attention for Improved Post-training Quantization in Large Sequence-to-sequence Speech Foundation Models}, editor = {Pesak, Krisztina}, doi = {10.21437/Interspeech.2024-2105}, pages = {4623 -- 4627}, year = {2024}, abstract = {This paper explores the improvement of post-training quantization (PTQ) after knowledge distillation in the Whisper speech foundation model family. We address the challenge of outliers in weights and activation tensors, known to impede quantization quality in transformer-based language and vision models. Extending this observation to Whisper, we demonstrate that these outliers are also present when transformer-based models are trained to perform automatic speech recognition, necessitating mitigation strategies for PTQ. We show that outliers can be reduced by a recently proposed gating mechanism in the attention blocks of the student model, enabling effective 8-bit quantization, and lower word error rates compared to student models without the gating mechanism in place.}, language = {en} } @inproceedings{RanzenbergerFreierReinoldetal.2024, author = {Ranzenberger, Thomas and Freier, Carolin and Reinold, Luca and Riedhammer, Korbinian and Schneider, Fabian and Simic, Christopher and Simon, Claudia and Freisinger, Steffen and Georges, Munir and Bocklet, Tobias}, title = {A Multidisciplinary Approach to AI-based self-motivated Learning and Teaching with Large Language Models}, series = {Proceedings of DELFI 2024}, booktitle = {Proceedings of DELFI 2024}, publisher = {Gesellschaft f{\"u}r Informatik e.V.}, address = {Bonn}, issn = {2944-7682}, doi = {10.18420/delfi2024_11}, pages = {133-140}, year = {2024}, abstract = {We present a learning experience platform that uses machine learning methods to support students and lecturers in self-motivated online learning and teaching processes. The platform is being developed as an agile open-source collaborative project supported by multiple universities and partners. The development is guided didactically, reviewed, and scientifically evaluated in several cycles. Transparency, data protection and the copyright compliant use of the system is a central part of the project. The system further employs large language models (LLMs). Due to privacy concerns, we utilize locally hosted LLM instances and explicitly do not rely on available cloud products. Students and lecturers can interact with an LLM-based chatbot in the current prototype. The AI-generated outputs contain cross-references to the current educational video's context, indicating if sections are based on the lectures context or world knowledge. We present the prototype and results of our qualitative evaluation from the perspective of lecturers and students.}, language = {en} } @article{EscobarGrisalesRiosUrregoBaumannetal.2024, author = {Escobar-Grisales, Daniel and R{\´i}os-Urrego, Cristian-David and Baumann, Ilja and Riedhammer, Korbinian and N{\"o}th, Elmar and Bocklet, Tobias and Garcia, Adolfo and Orozco-Arroyave, Juan rafael}, title = {It's Time to Take Action: Acoustic Modeling of Motor Verbs to Detect Parkinson's Disease}, doi = {10.21437/Interspeech.2024-2205}, year = {2024}, abstract = {Pre-trained models generate speech representations that are used in different tasks, including the automatic detection of Parkinson's disease (PD). Although these models can yield high accuracy, their interpretation is still challenging. This paper used a pre-trained Wav2vec 2.0 model to represent speech frames of 25ms length and perform a frame-by-frame discrimination between PD patients and healthy control (HC) subjects. This fine granularity prediction enabled us to identify specific linguistic segments with high discrimination capability. Speech representations of all produced verbs were compared w.r.t. nouns and the first ones yielded higher accuracies. To gaina deeper understanding of this pattern, representations of motor and non-motor verbs were compared and the first ones yielded better results, with accuracies of around 83\% in an independent test set. These findings support well-established neurocognitive models about action-related language highlighted as key drivers of PD. Index Terms: computational paralinguistics, interpretability of pre-trained models, action verbs, Parkinson's disease}, language = {en} }