@inproceedings{RanzenbergerFreierReinoldetal.2024, author = {Ranzenberger, Thomas and Freier, Carolin and Reinold, Luca and Riedhammer, Korbinian and Schneider, Fabian and Simic, Christopher and Simon, Claudia and Freisinger, Steffen and Georges, Munir and Bocklet, Tobias}, title = {A Multidisciplinary Approach to AI-based self-motivated Learning and Teaching with Large Language Models}, series = {Proceedings of DELFI Workshops 2024}, booktitle = {Proceedings of DELFI Workshops 2024}, publisher = {Gesellschaft f{\"u}r Informatik e.V.}, doi = {10.18420/delfi2024_11}, year = {2024}, abstract = {We present a learning experience platform that uses machine learning methods to support students and lecturers in self-motivated online learning and teaching processes. The platform is being developed as an agile open-source collaborative project supported by multiple universities and partners. The development is guided didactically, reviewed, and scientifically evaluated in several cycles. Transparency, data protection and the copyright compliant use of the system is a central part of the project. The system further employs large language models (LLMs). Due to privacy concerns, we utilize locally hosted LLM instances and explicitly do not rely on available cloud products. Students and lecturers can interact with an LLM-based chatbot in the current prototype. The AI-generated outputs contain cross-references to the current educational video's context, indicating if sections are based on the lectures context or world knowledge. We present the prototype and results of our qualitative evaluation from the perspective of lecturers and students.}, language = {en} } @inproceedings{WagnerBaumannRanzenbergeretal.2024, author = {Wagner, Dominik and Baumann, Ilja and Ranzenberger, Thomas and Riedhammer, Korbinian and Bocklet, Tobias}, title = {Personalizing Large Sequence-to-Sequence Speech Foundation Models With Speaker Representations}, doi = {10.1109/SLT61566.2024.10832252}, pages = {1-6}, year = {2024}, abstract = {We present a method to personalize large transformer-based encoderdecoder speech foundation models without the need for changes in the underlying model structure or training from scratch. This is achieved by projecting speaker-specific information into the latent space of the transformer decoder via a small neural network and learning to process the speaker information along with domainspecific information via parameter-efficient finetuning. We use this method to improve the automatic speech recognition results of spoken academic German and English. Our approach yields average relative word error rate (WER) improvements of approximately 29\% on German academic speech and 25\% on English academic speech. It also translates well to conversational speech, achieving relative WER improvements of up to 36\%, and demonstrates modest gains of up to 5\% on read speech. Moreover, we observe that incorporating utterances from the recent past as personalization context yields the most significant overall improvements and that changes in voice characteristics resulting from prolonged speaking have a minimal effect on the personalization quality of academic lectures.}, language = {en} } @inproceedings{RanzenbergerBockletFreisingeretal.2024, author = {Ranzenberger, Thomas and Bocklet, Tobias and Freisinger, Steffen and Georges, Munir and Glockner, Kevin and Herygers, Aaricia and Riedhammer, Korbinian and Schneider, Fabian and Simic, Christopher and Zakaria, Khabbab}, title = {EXTENDING HANS: LARGE LANGUAGE MODELS FOR QUESTION ANSWERING, SUMMARIZATION, AND TOPIC SEGMENTATION IN AN ML-BASED LEARNING EXPERIENCE PLATFORM}, series = {Elektronische Sprachsignalverarbeitung 2024, Tagungsband der 35. Konferenz, Regensburg, 6.-8. M{\"a}rz 2024}, booktitle = {Elektronische Sprachsignalverarbeitung 2024, Tagungsband der 35. Konferenz, Regensburg, 6.-8. M{\"a}rz 2024}, publisher = {TUPress}, address = {Dresden}, isbn = {978-3-95908-325-6}, pages = {219-224}, year = {2024}, abstract = {Abstract: The use of chatbots based on large language models (LLMs) and their impact on society are influencing our learning experience platform Hochschul Assistenz-System (HAnS). HAnS uses machine learning (ML) methods to support students and lecturers in the online learning and teaching processes [1]. This paper introduces LLM-based features available in HAnS which are using the transcript of our improved Automatic Speech Recognition (ASR) pipeline with an average transcription duration of 45 seconds and an average word error rate (WER) of 6.66\% on over 8 hours of audio data of 7 lecture videos. A LLM-based chatbot could be used to answer questions on the lecture content as the ASR transcript is provided as context. The summarization and topic segmentation uses the LLM to improve our learning experience platform. We generate multiple choice questions using the LLM and the ASR transcript as context during playback in a period of 3 minutes and display them in the HAnS frontend}, language = {en} } @inproceedings{RanzenbergerFreierReinoldetal.2024, author = {Ranzenberger, Thomas and Freier, Carolin and Reinold, Luca and Riedhammer, Korbinian and Schneider, Fabian and Simic, Christopher and Simon, Claudia and Freisinger, Steffen and Georges, Munir and Bocklet, Tobias}, title = {A Multidisciplinary Approach to AI-based self-motivated Learning and Teaching with Large Language Models}, series = {Proceedings of DELFI 2024}, booktitle = {Proceedings of DELFI 2024}, publisher = {Gesellschaft f{\"u}r Informatik e.V.}, address = {Bonn}, issn = {2944-7682}, doi = {10.18420/delfi2024_11}, pages = {133-140}, year = {2024}, abstract = {We present a learning experience platform that uses machine learning methods to support students and lecturers in self-motivated online learning and teaching processes. The platform is being developed as an agile open-source collaborative project supported by multiple universities and partners. The development is guided didactically, reviewed, and scientifically evaluated in several cycles. Transparency, data protection and the copyright compliant use of the system is a central part of the project. The system further employs large language models (LLMs). Due to privacy concerns, we utilize locally hosted LLM instances and explicitly do not rely on available cloud products. Students and lecturers can interact with an LLM-based chatbot in the current prototype. The AI-generated outputs contain cross-references to the current educational video's context, indicating if sections are based on the lectures context or world knowledge. We present the prototype and results of our qualitative evaluation from the perspective of lecturers and students.}, language = {en} } @inproceedings{FreisingerSeebergerRanzenbergeretal.2025, author = {Freisinger, Steffen and Seeberger, Philipp and Ranzenberger, Thomas and Bocklet, Tobias and Riedhammer, Korbinian}, title = {Towards Multi-Level Transcript Segmentation: LoRA Fine-Tuning for Table-of-Contents Generation}, series = {Interspeech 2025}, booktitle = {Interspeech 2025}, publisher = {ISCA}, address = {ISCA}, issn = {2958-1796}, doi = {10.21437/Interspeech.2025-2792}, pages = {276 -- 280}, year = {2025}, abstract = {Segmenting speech transcripts into thematic sections benefits both downstream processing and users who depend on written text for accessibility. We introduce a novel approach to hierarchical topic segmentation in transcripts, generating multi-level tables of contents that capture both topic and subtopic boundaries. We compare zero-shot prompting and LoRA fine-tuning on large language models, while also exploring the integration of high-level speech pause features. Evaluations on English meeting recordings and multilingual lecture transcripts (Portuguese, German) show significant improvements over established topic segmentation baselines. Additionally, we adapt a common evaluation measure for multi-level segmentation, taking into account all hierarchical levels within one metric.}, language = {en} } @article{RanzenbergerBaumannBayerletal.2025, author = {Ranzenberger, Thomas and Baumann, Ilja and Bayerl, Sebastian and Wagner, Dominik and Bocklet, Tobias and Riedhammer, Korbinian}, title = {Evaluation of recognition errors of hybrid and transformer-based ASR systems in German video lectures}, series = {Studientexte zur Sprachkommunikation: Elektronische Sprachsignalverarbeitung 2025 - Book}, journal = {Studientexte zur Sprachkommunikation: Elektronische Sprachsignalverarbeitung 2025 - Book}, publisher = {ESSV 2025}, address = {Halle, Deutschland}, pages = {101-108}, year = {2025}, abstract = {We analyze different errors in speech recognition systems, focusing on consecutive insertions and deletions, known as hallucinations and elisions in transformer-based end-to-end automatic speech recognition (ASR) systems. We compare errors from a TDNN-HMM, and whisper-based models on English and German spontaneous speech. Based on a human annotated subset of German lecture videos, we investigate whether these blocks of deletions affect the semantics of the utterance. Whisper performs best and preserves the meaning in 90\% of the annotated error segments even containing consecutive deletions on this subset. We analyze the word error rate and do further analysis of errors using natural language processing to detect lemmatization errors, compound word errors, and out-of-vocabulary words. We discuss possible reasons and mitigations.}, language = {en} }