@article{MeyerSickendiekHusseinBaumann, author = {Meyer-Sickendiek, Burkhard and Hussein, Hussein and Baumann, Timo}, title = {Towards the Creation of a Poetry Translation Mapping System}, series = {Archives of Data Science, Series A}, volume = {5}, journal = {Archives of Data Science, Series A}, number = {1}, publisher = {Towards the Creation of a Poetry Translation Mapping System}, issn = {2363-9881}, doi = {10.5445/KSP/1000087327/21}, pages = {1 -- 15}, abstract = {The translation of poetry is a complex, multifaceted challenge: the translated text should communicate the same meaning, similar metaphoric expressions, and also match the style and prosody of the original poem. Research on machine poetry translation is existing since 2010, but for four reasons it is still rather insufficient: 1. The few approaches existing completely lack any knowledge about current developments in both lyric theory and translation theory. 2. They are based on very small datasets. 3. They mostly ignored the neural learning approach that superseded the long-standing dominance of phrase-based approaches within machine translation. 4. They have no concept concerning the pragmatic function of their research and the resulting tools. Our paper describes how to improve the existing research and technology for poetry translations in exactly these four points. With regards to 1) we will describe the "Poetics of Translation". With regards to 2) we will introduce the Worlds largest corpus for poetry translations from lyrikline. With regards to 3) we will describe first steps towards a neural machine translation of poetry. With regards to 4) we will describe first steps towards the development of a poetry translation mapping system.}, language = {en} } @misc{WardBaumannKarkhedkaretal., author = {Ward, Nigel and Baumann, Timo and Karkhedkar, Shreyas and Novick, David}, title = {Dynamic control of voice codec data rate}, abstract = {A method, system, and computer-usable non-transitory storage device for dynamic voice codec adaptation are disclosed. The voice codec adapts in real time to devote more bits to audio quality when it is most needed, and fewer bits to less important parts of utterances are disclosed. Dialog knowledge is utilized for compression opportunities to adjust the bitrate moment-by-moment, based on the inferred value of each frame. Frame importance and appropriate transmission fidelity is predicted based on prosodic features and models of dialog dynamics. This technique provides the same communications quality with less spectrum needs, fewer antennas, and less battery drain.}, language = {en} } @inproceedings{JohannssenBiemannRemusetal., author = {Johannßen, Dirk and Biemann, Chris and Remus, Steffen and Baumann, Timo and Scheffer, David}, title = {GermEval 2020 Task 1 on the Classification and Regression of Cognitive and Motivational style from Text}, series = {Proceedings of the 5th SwissText \& 16th KONVENS Joint Conference 2020}, booktitle = {Proceedings of the 5th SwissText \& 16th KONVENS Joint Conference 2020}, address = {Zurich, Switzerland}, pages = {1 -- 10}, abstract = {This paper describes the tasks, databases, baseline systems, and summarizes submissions and results for the GermEval 2020 Shared Task 1 on the Classification and Regression of Cognitive and Motivational Style from Text. This shared task is divided into two subtasks, a regression task, and a classification task. Subtask 1 asks participants to reproduce a ranking of students based on average aptitude indicators such as different high school grades and different IQ scores. The second subtask aims to classify so-called implicit motives, which are projective testing procedures that can reveal unconscious desires. Besides five implicit motives, the target labels of Subtask 2 also contain one of six levels that describe the type of self-regulation when acting out a motive, which makes this task a multiclass-classification with 30 target labels. 3 participants submitted multiple systems. Subtask 1 was solved (best r = .3701) mainly with non-neural systems and statistical language representations, submissions for Subtask 2 utilized neural approaches and word embeddings (best macro F1 = 70.40). Not only were the tasks solvable, analyses by the participants even showed connections to the implicit psychometrics theory and behavioral observations made by psychologists. This paper describes the tasks, databases, baseline systems, and summarizes submissions and results for the GermEval 2020 Shared Task 1 on the Classification and Regression of Cognitive and Motivational Style from Text. This shared task is divided into two subtasks, a regression task, and a classification task. Subtask 1 asks participants to reproduce a ranking of students based on average aptitude indicators such as different high school grades and different IQ scores. The second subtask aims to classify so-called implicit motives, which are projective testing procedures that can reveal unconscious desires. Besides five implicit motives, the target labels of Subtask 2 also contain one of six levels that describe the type of self-regulation when acting out a motive, which makes this task a multiclass-classification with 30 target labels. 3 participants submitted multiple systems. Subtask 1 was solved (best r =.3701) mainly with non-neural systems and statistical language representations, submissions for Subtask 2 utilized neural approaches and word embeddings (best macro F1 = 70.40). Not only were the tasks solvable, analyses by the participants even showed connections to the implicit psychometrics theory and behavioral observations made by psychologists.}, language = {en} } @inproceedings{HusseinMeyerSickendiekBaumann, author = {Hussein, Hussein and Meyer-Sickendiek, Burkhard and Baumann, Timo}, title = {Free Verse and Beyond: How to Classify Post-modern Spoken Poetry}, series = {Proceedings of Speech Prosody: Tokyo, Japan, 25-28 May 2020}, booktitle = {Proceedings of Speech Prosody: Tokyo, Japan, 25-28 May 2020}, publisher = {ISCA}, doi = {10.21437/SpeechProsody.2020-141}, pages = {690 -- 694}, abstract = {This paper presents the classification of rhythmical patterns detected in post-modern spoken poetry by means of machine learning algorithms that use manually engineered features or automatically learnt representations. We used the world's largest corpus of spoken poetry from our partner lyrikline. We identified nine rhythmical patterns within a spectrum raging from a more fluent to a more disfluent poetic style. The text data analyzed by a statistical parser. Prosodic features of rhythmical patterns are identified by using the parser information. For the classification of rhythmical patterns, we used a neural networks-based approach which use text, audio, and pause information between poetic lines as features. Different combinations of features as well as the integration of feature engineering in the neural networks-based approach are tested. We compared the performance of both approaches (feature-based and neural network-based) using combinations of different features. The results show - by using the weighted average of f-measure for the evaluation - that the neural networks-based approach performed much better in classification of rhythmical patterns. The important improvement of the classification results lies in the use of the audio information. The integration of feature engineering in the neural networks-based approach yielded a very small result improvement.}, language = {en} } @inproceedings{BaumannMeyerSickendiek, author = {Baumann, Timo and Meyer-Sickendiek, Burkhard}, title = {Deep Learning meets Post-modern Poetry}, series = {TwinTalks 2020: Understanding and Facilitating Collaboration in Digital Humanities 2020, proceedings of the Twin Talks 2 and 3 workshops at DHN 2020 and DH 2020, Ottawa Canada and Riga Latvia, July 23 and October 20, 2020}, booktitle = {TwinTalks 2020: Understanding and Facilitating Collaboration in Digital Humanities 2020, proceedings of the Twin Talks 2 and 3 workshops at DHN 2020 and DH 2020, Ottawa Canada and Riga Latvia, July 23 and October 20, 2020}, editor = {Krauwer, Steven and Fišer, Darja}, publisher = {RWTH Aachen}, issn = {1613-0073}, pages = {30 -- 36}, abstract = {We summarize our project Rhythmicalizer in which we analyze a corpus of post-modern poetry in a combination of qualitative hermeneutical and computational methods, as we have run the project over the course of the past three years (and preparing it for some time before that). Interdisciplinary work is always challenging and we here focus on some of the highlights of our collaboration.}, language = {en} } @inproceedings{Baumann, author = {Baumann, Timo}, title = {How a Listener Influences the Speaker}, series = {Proceedings of Speech Prosody: Tokyo, Japan, 25-28 May 2020}, booktitle = {Proceedings of Speech Prosody: Tokyo, Japan, 25-28 May 2020}, publisher = {ISCA}, doi = {10.21437/SpeechProsody.2020-198}, pages = {970 -- 974}, abstract = {Listeners typically provide feedback while listening to a speaker in conversation and thereby engage in the co-construction of the interaction. We analyze the influence of the listener on the speaker by investigating how her verbal feedback signals help in modeling the speaker's language. We find that feedback from the listener may help in modeling the speaker's language, whether through the listener's feedback as transcribed, or the acoustic signal directly. We find the largest positive effects for end of sentence as well as for pauses mid-utterance, but also effects that indicate we successfully model elaborations of ongoing utterances that may result from the presence or absence of listener feedback.}, language = {en} } @article{BaumannKoehnHennig, author = {Baumann, Timo and K{\"o}hn, Arne and Hennig, Felix}, title = {The Spoken Wikipedia Corpus collection: Harvesting, alignment and an application to hyperlistening}, series = {Language Resources and Evaluation}, volume = {53}, journal = {Language Resources and Evaluation}, number = {2}, publisher = {Springer Nature}, doi = {10.1007/s10579-017-9410-y}, pages = {303 -- 329}, abstract = {Spoken corpora are important for speech research, but are expensive to create and do not necessarily reflect (read or spontaneous) speech 'in the wild'. We report on our conversion of the preexisting and freely available Spoken Wikipedia into a speech resource. The Spoken Wikipedia project unites volunteer readers of Wikipedia articles. There are initiatives to create and sustain Spoken Wikipedia versions in many languages and hence the available data grows over time. Thousands of spoken articles are available to users who prefer a spoken over the written version. We turn these semi-structured collections into structured and time-aligned corpora, keeping the exact correspondence with the original hypertext as well as all available metadata. Thus, we make the Spoken Wikipedia accessible for sustainable research. We present our open-source software pipeline that downloads, extracts, normalizes and text-speech aligns the Spoken Wikipedia. Additional language versions can be exploited by adapting configuration files or extending the software if necessary for language peculiarities. We also present and analyze the resulting corpora for German, English, and Dutch, which presently total 1005 h and grow at an estimated 87 h per year. The corpora, together with our software, are available via http://islrn.org/resources/684-927-624-257-3/. As a prototype usage of the time-aligned corpus, we describe an experiment about the preferred modalities for interacting with information-rich read-out hypertext. We find alignments to help improve user experience and factual information access by enabling targeted interaction.}, language = {en} } @inproceedings{SabooBaumann, author = {Saboo, Ashutosh and Baumann, Timo}, title = {Integration of Dubbing Constraints into Machine Translation}, series = {Proceedings of the Fourth Conference on Machine Translation, Volume 1: Research Papers}, booktitle = {Proceedings of the Fourth Conference on Machine Translation, Volume 1: Research Papers}, editor = {Bojar, Ondřej and Chatterjee, Rajen and Federmann, Christian and Fishel, Mark and Graham, Yvette and Haddow, Barry and Huck, Matthias and Yepes, Antonio Jimeno and Koehn, Philipp and Martins, Andr{\´e} and Monz, Christof and Negri, Matteo and N{\´e}v{\´e}ol, Aur{\´e}lie and Neves, Mariana and Post, Matt and Turchi, Marco and Verspoor, Karin}, publisher = {Association for Computational Linguistics}, address = {Stroudsburg, PA, USA}, doi = {10.18653/v1/W19-5210}, pages = {94 -- 101}, abstract = {Translation systems aim to perform a meaning-preserving conversion of linguistic material (typically text but also speech) from a source to a target language (and, to a lesser degree, the corresponding socio-cultural contexts). Dubbing, i.e., the lip-synchronous translation and revoicing of speech adds to this constraints about the close matching of phonetic and resulting visemic synchrony characteristics of source and target material. There is an inherent conflict between a translation's meaning preservation and 'dubbability' and the resulting trade-off can be controlled by weighing the synchrony constraints. We introduce our work, which to the best of our knowledge is the first of its kind, on integrating synchrony constraints into the machine translation paradigm. We present first results for the integration of synchrony constraints into encoder decoder-based neural machine translation and show that considerably more 'dubbable' translations can be achieved with only a small impact on BLEU score, and dubbability improves more steeply than BLEU degrades.}, language = {en} } @inproceedings{HusseinMeyerSickendiekBaumann, author = {Hussein, Hussein and Meyer-Sickendiek, Burkhard and Baumann, Timo}, title = {Identification of Concrete Poetry within a Modern-Poetry Corpus using Neural Networks}, series = {Proceedings of the Quantitative Approaches to Versification Conference}, booktitle = {Proceedings of the Quantitative Approaches to Versification Conference}, address = {Prague, Czech Republic}, pages = {95 -- 104}, abstract = {This work aims to discern the poetics of concrete poetry by using a corpus-based classification focusing on the two most important techniques used within concrete poetry: semantic decomposition and syntactic permutation. We demonstrate how to identify concrete poetry in modern and postmodern free verse. A class contrasting to concrete poetry is defined on the basis of poems with complete and correct sentences. We used the data from lyrikline, which contain both the written as well as the spoken form of poems as read by the original author. We explored two approaches for the identification of concrete poetry. The first is based on the definition of concrete poetry in literary theory by the extraction of various types of features derived from a parser, such as verb, noun, comma, sentence ending, conjunction, and asemantic material. The second is a neural network-based approach, which is theoretically less informed by human insight, as it does not have access to features established by scholars. This approach used the following inputs: textual information and the spoken recitation of poetic lines as well as information about pauses between lines. The results based on the neural network are more accurate than the feature-based approach. The best results, calculated by the weighted F-measure, for the classification of concrete poetry vis-{\`a}-vis the contrasting class is 0.96}, language = {en} } @incollection{Baumann, author = {Baumann, Timo}, title = {Ranking and Comparing Speakers Based on Crowdsourced Pairwise Listener Ratings}, series = {Voice attractiveness: Studies on Sexy, Likable, and Charismatic Speakers}, booktitle = {Voice attractiveness: Studies on Sexy, Likable, and Charismatic Speakers}, editor = {Weiss, Benjamin and Trouvain, J{\"u}rgen and Barkat-Defradas, M{\´e}lissa and Ohala, John J.}, publisher = {Springer}, address = {Singapore}, isbn = {978-981-15-6626-4}, doi = {10.1007/978-981-15-6627-1_14}, pages = {263 -- 279}, abstract = {Speech quality and likability is a multi-faceted phenomenon consisting of a combination of perceptory features that cannot easily be computed nor weighed automatically. Yet, it is often easy to decide which of two voices one likes better, even though it would be hard to describe why, or to name the underlying basic perceptory features. Although likability is inherently subjective and individual preferences differ, generalizations are useful and there is often a broad intersubjective consensus about whether one speaker is more likeable than another. We present a methodology to efficiently create a likability ranking for many speakers from crowdsourced pairwise likability ratings which focuses manual rating effort on pairs of similar quality using an active sampling technique. Using this methodology, we collected pairwise likability ratings for many speakers (>220) from many raters (>160). We analyze listener preferences by correlating the resulting ranking with various acoustic and prosodic features. We also present a neural network that is able to model the complexity of listener preferences and the underlying temporal evolution of features. The recurrent neural network achieves remarkably high performance in estimating the pairwise decisions and an ablation study points toward the criticality of modeling temporal aspects in speech quality assessment.}, language = {en} } @inproceedings{TsaiBaumannPecuneetal., author = {Tsai, Vivian and Baumann, Timo and Pecune, Florian and Cassell, Justine}, title = {Faster Responses Are Better Responses: Introducing Incrementality into Sociable Virtual Personal Assistants}, series = {9th International Workshop on Spoken Dialogue System Technology}, volume = {579}, booktitle = {9th International Workshop on Spoken Dialogue System Technology}, editor = {D'Haro, Luis Fernando and Banchs, Rafael E. and Li, Haizhou}, publisher = {Springer Singapore}, address = {Singapore}, isbn = {978-981-13-9442-3}, issn = {1876-1119}, doi = {10.1007/978-981-13-9443-0_10}, pages = {111 -- 118}, abstract = {Speech-based interactive systems, such as virtual personal assistants, inevitably use complex architectures, with a multitude of modules working in series (or less often in parallel) to perform a task (e.g., giving personalized movie recommendations via dialog). Add modules for evoking and sustaining sociability with the user and the accumulation of processing latencies through the modules results in considerable turn-taking delays. We introduce incremental speech processing into the generation pipeline of the system to overcome this challenge with only minimal changes to the system architecture, through partial underspecification that is resolved as necessary. A user study with a sociable movie recommendation agent objectively diminishes turn-taking delays; furthermore, users not only rate the incremental system as more responsive, but also rate its recommendation performance as higher.}, language = {en} } @inproceedings{NayakBaumannBhattacharyaetal., author = {Nayak, Shravan and Baumann, Timo and Bhattacharya, Supratik and Karakanta, Alina and Negri, Matteo and Turchi, Marco}, title = {See me Speaking? Differentiating on Whether Words are Spoken On Screen or Off to Optimize Machine Dubbing}, series = {ICMI '20 Companion: Companion Publication of the 2020 International Conference on Multimodal Interaction, 25.10.2020 - 29.10.2020; Virtual Event Netherlands}, booktitle = {ICMI '20 Companion: Companion Publication of the 2020 International Conference on Multimodal Interaction, 25.10.2020 - 29.10.2020; Virtual Event Netherlands}, editor = {Truong, Khiet}, publisher = {Association for Computing Machinery}, address = {New York,NY,United States}, isbn = {9781450380027}, doi = {10.1145/3395035.3425640}, pages = {130 -- 134}, abstract = {Dubbing is the art of finding a translation from a source into a target language that can be lip-synchronously revoiced, i. e., that makes the target language speech appear as if it was spoken by the very actors all along. Lip synchrony is essential for the full-fledged reception of foreign audiovisual media, such as movies and series, as violated constraints of synchrony between video (lips) and audio (speech) lead to cognitive dissonance and reduce the perceptual quality. Of course, synchrony constraints only apply to the translation when the speaker's lips are visible on screen. Therefore, deciding whether to apply synchrony constraints requires an automatic method for detecting whether an actor's lips are visible on screen for a given stretch of speech or not. In this paper, we attempt, for the first time, to classify on- from off-screen speech based on a corpus of real-world television material that has been annotated word-by-word for the visibility of talking lips on screen. We present classification experiments in which we classify}, language = {en} } @inproceedings{GeislingerMildeBaumannetal., author = {Geislinger, Robert and Milde, Benjamin and Baumann, Timo and Biemann, Chris}, title = {Live Subtitling for BigBlueButton with Open-Source Software}, series = {INTERSPEECH 2021: (Show \& Tell Contribution), 30 August - 3 September, 2021, Brno, Czechia}, booktitle = {INTERSPEECH 2021: (Show \& Tell Contribution), 30 August - 3 September, 2021, Brno, Czechia}, address = {Brno, Czechia}, pages = {3319 -- 3320}, abstract = {We present an open source plugin for live subtitling in the popular open source video conferencing software BigBlueBut-ton. Our plugin decodes each speaker's audio stream separately and in parallel, thereby obliviating the need for speaker di-arization and seamlessly handling overlapped talk. Any Kaldi-compatible nnet3 model can be used with our plugin and we demonstrate it using freely available TDNN-HMM-based ASR models for English and German. Our subtitles can be used as they are (e.g., in loud environments) or can form the basis for further NLP processes. Our tool can also simplify the collection of remotely recorded multi-party dialogue corpora.}, language = {en} } @inproceedings{BaumannHusseinMeyerSickendieketal., author = {Baumann, Timo and Hussein, Hussein and Meyer-Sickendiek, Burkhard and Elbeshausen, Jasper}, title = {A Tool for Human-in-the-Loop Analysis and Exploration of (not only) Prosodic Classifications for Post-modern Poetry}, series = {Informatik 2019 : 50 Jahre Gesellschaft f{\"u}r Informatik Workshop-Beitr{\"a}ge Fachtagung vom 23.-26. September 2019 in Kassel}, booktitle = {Informatik 2019 : 50 Jahre Gesellschaft f{\"u}r Informatik Workshop-Beitr{\"a}ge Fachtagung vom 23.-26. September 2019 in Kassel}, editor = {Draude, Claude and Lange, Martin and Sick, Bernhard}, publisher = {Gesellschaft f{\"u}r Informatik}, address = {Bonn}, isbn = {978-3-88579-689-3}, pages = {151 -- 156}, abstract = {Data-based analyses are becoming more and more common in the Digital Humanities and tools are needed that focus human efforts on the most interesting and important aspects of exploration, analysis and annotation by using active machine learning techniques. We present our ongoing work on a tool that supports classification tasks for spoken documents (in our case: read-out post-modern poetry) using a neural networks-based classification backend and a web-based exploration and classification environment.}, language = {en} } @inproceedings{KarakantaBhattacharyaNayaketal., author = {Karakanta, Alina and Bhattacharya, Supratik and Nayak, Shravan and Baumann, Timo and Negri, Matteo and Turchi, Marco}, title = {The Two Shades of Dubbing in Neural Machine Translation}, series = {Proceedings of the 28th International Conference on Computational Linguistics (COLING 2020): Barcelona, Spain (Online)}, booktitle = {Proceedings of the 28th International Conference on Computational Linguistics (COLING 2020): Barcelona, Spain (Online)}, editor = {Scott, Donia and Bel, Nuria and Zong, Chengqing}, publisher = {International Committee on Computational Linguistics}, address = {Stroudsburg, PA, USA}, doi = {10.18653/v1/2020.coling-main.382}, pages = {4327 -- 4333}, abstract = {Dubbing has two shades; synchronisation constraints are applied only when the actor's mouth is visible on screen, while the translation is unconstrained for off-screen dubbing. Consequently, different synchronisation requirements, and therefore translation strategies, are applied depending on the type of dubbing. In this work, we manually annotate an existing dubbing corpus (Heroes) for this dichotomy. We show that, even though we did not observe distinctive features between on- and off-screen dubbing at the textual level, on-screen dubbing is more difficult for MT (-4 BLEU points). Moreover, synchronisation constraints dramatically decrease translation quality for off-screen dubbing. We conclude that, distinguishing between on-screen and off-screen dubbing is necessary for determining successful strategies for dubbing-customised Machine Translation.}, language = {en} } @inproceedings{FischerNaikLangedijketal., author = {Fischer, Kerstin and Naik, Lakshadeep and Langedijk, Rosalyn Melissa and Baumann, Timo and Jel{\´i}nek, Matouš and Palinko, Oskar}, title = {Initiating Human-Robot Interactions Using Incremental Speech Adaptation}, series = {HRI '21 Companion, Companion of the 2021 ACM/IEEE International Conference on Human-Robot Interaction: March 8 - 11, 2021, Boulder CO, USA}, booktitle = {HRI '21 Companion, Companion of the 2021 ACM/IEEE International Conference on Human-Robot Interaction: March 8 - 11, 2021, Boulder CO, USA}, editor = {Bethel, Cindy and Paiva, Ana}, publisher = {Association for Computing Machinery}, address = {New York, NY}, isbn = {9781450382908}, doi = {10.1145/3434074.3447205}, pages = {421 -- 425}, abstract = {In this paper, we present a study in which a robot initiates interactions with people passing by in an in-the-wild scenario. The robot adapts the loudness of its voice dynamically to the distance of the respective person approached, thus indicating who it is talking to. It furthermore tracks people based on information on body orientation and eye gaze and adapts the text produced based on people's distance autonomously. Our study shows that the adaptation of the loudness of its voice is perceived as personalization by the participants and that the likelihood that they stop by and interact with the robot increases when the robot incrementally adjusts its behavior.}, language = {en} } @inproceedings{SahaNayakBaumann, author = {Saha, Debjoy and Nayak, Shravan and Baumann, Timo}, title = {Merkel Podcast Corpus: A Multimodal Dataset Compiled from 16 Years of Angela Merkel's Weekly Video Podcasts}, series = {Proceedings of the 13th Conference on Language Resources and Evaluation (LREC 2022), Marseille, 20-25 June 2022}, booktitle = {Proceedings of the 13th Conference on Language Resources and Evaluation (LREC 2022), Marseille, 20-25 June 2022}, organization = {European Language Resources Association (ELRA)}, pages = {2536 -- 2540}, abstract = {We introduce the Merkel Podcast Corpus, an audio-visual-text corpus in German collected from 16 years of (almost) weekly Internet podcasts of former German chancellor Angela Merkel. To the best of our knowledge, this is the first single speaker corpus in the German language consisting of audio, visual and text modalities of comparable size and temporal extent. We describe the methods used with which we have collected and edited the data which involves downloading the videos, transcripts and other metadata, forced alignment, performing active speaker recognition and face detection to finally curate the single speaker dataset consisting of utterances spoken by Angela Merkel. The proposed pipeline is general and can be used to curate other datasets of similar nature, such as talk show contents. Through various statistical analyses and applications of the dataset in talking face generation and TTS, we show the utility of the dataset. We argue that it is a valuable contribution to the research community, in particular, due to its realistic and challenging material at the boundary between prepared and spontaneous speech. Accepted at LREC 2022}, language = {en} } @inproceedings{BaumannSchlangen, author = {Baumann, Timo and Schlangen, David}, title = {The INPROTK 2012 release}, series = {Sprachkommunikation 2012 : Beitr{\"a}ge zur 10. ITG-Fachtagung vom 26. bis 28. September 2012 in Braunschweig}, booktitle = {Sprachkommunikation 2012 : Beitr{\"a}ge zur 10. ITG-Fachtagung vom 26. bis 28. September 2012 in Braunschweig}, editor = {Fingscheidt, T.}, publisher = {VDE-Verl}, address = {Berlin ; Offenbach}, isbn = {978-3-8007-3455-9}, pages = {147 -- 150}, abstract = {We describe the 2012 release of INPROTK1, our "Incremental Processing Toolkit" which combines a powerful and extensible architecture for incremental processing with components for incremental speech recognition and, new to this release, incremental speech synthesis. These components work domainindependently; we also provide example implementations of higher-level components such as natural language understanding and dialogue management that are somewhat more tied to a particular domain. The toolkit is accompanied by evaluation tools for analysing timing behaviour, and we highlight some timing results on conversational speech input in this paper. We offer our toolkit to foster research in this new and exciting area, which promises to help increase the naturalness of behaviours that can be modelled in such systems.}, language = {en} } @misc{BaumannGeiserMenzeletal., author = {Baumann, Timo and Geiser, Dorothee and Menzel, Wolfgang and Mohr, Mario and Neef, Svenja and Nykamp, S{\"o}ren and Rokita, Nils}, title = {Concurrent Sub-turn Interaction Specification and Dialogue Management with an Application to Interactive Storytelling}, series = {GSCL Workshop Gesprochene Sprache und Sprachverarbeitung (GSS), Darmstadt, Germany, 2013}, journal = {GSCL Workshop Gesprochene Sprache und Sprachverarbeitung (GSS), Darmstadt, Germany, 2013}, address = {Darmstadt, Germany}, abstract = {Conventional dialogue management centers around an interaction style that is best described as a ping-pong game, with full turns being the units at which speech is delivered (and expected) by the system, which greatly simplifies the interaction management, delivery and understanding components of the system. While the resulting mode of interaction works well for task-based systems, it is insufficient for more conversational interaction styles, where content is delivered and grounded in units finer than full turns (Poesio and Traum 1997) and where turns are delivered concurrently by both interlocutors and hence overlap more frequently. One domain with particularly frequent overlapping contributions is interactive storytelling: a storyteller that is responsive to listeners will integrate their feedback immediately while still speaking a current contribution, and listener's remarks or propositions regarding the story will typically be uttered immediately when the related content is delivered. We present our work on a dialogue manager that leverages recent advances in incremental speech delivery and reception (Baumann 2013) to provide for an interactive and concurrent storytelling experience. Our system uses an interaction graph, which uses a word-by-word granularity and allows to specify for individual stretches of speech where (and with what content) users may interrupt/comment, how this is interpreted, and how it is integrated into the storytelling process, thus providing for in-utterance alternatives to be spoken even without audibly interrupting the system's ongoing utterance. The system's interaction graph is specified in an XML language and may be hand-crafted by a story designer but can also be automatically generated. In our current system, speech recognition results are interpreted only when the user utterance is finished; however, we plan to integrate incremental speech recognition and understanding capabilities, and to immediately react to the start of user contributions.}, language = {en} } @inproceedings{SchenkeBaumann, author = {Schenke, Diana Marie and Baumann, Timo}, title = {Controlled Diversity: Length-optimized Natural Language Generation}, series = {Proceedings of the ISCA/ITG Workshop on Diversity in Large Speech and Language Models, February 20, 2025 in Berlin, Germany}, booktitle = {Proceedings of the ISCA/ITG Workshop on Diversity in Large Speech and Language Models, February 20, 2025 in Berlin, Germany}, editor = {M{\"o}ller, Sebastian and Knoeferle, Pia and Schulte, Britta and Feldhus, Nils}, doi = {10.48550/arXiv.2502.19347}, url = {http://nbn-resolving.de/urn:nbn:de:bvb:898-opus4-80534}, pages = {8}, abstract = {LLMs are not generally able to adjust the length of their outputs based on strict length requirements, a capability that would improve their usefulness in applications that require adherence to diverse user and system requirements. We present an approach to train LLMs to acquire this capability by augmenting existing data and applying existing fine-tuning techniques, which we compare based on the trained models' adherence to the length requirement and overall response quality relative to the baseline model. Our results demonstrate that these techniques can be successfully applied to train LLMs to adhere to length requirements, with the trained models generating texts which better align to the length requirements. Our results indicate that our method may change the response quality when using training data that was not generated by the baseline model. This allows simultaneous alignment to another training objective in certain scenarios, but is undesirable otherwise. Training on a dataset containing the model's own responses eliminates this issue.}, language = {en} } @unpublished{AmanpourBaumannHofmann, author = {Amanpour, Ali and Baumann, Timo and Hofmann, Ulrich G.}, title = {Optimizing Neural Data Analysis: Determining Minimum Recording Length for Unambigous Signal Processing}, doi = {10.35096/othr/pub-8060}, url = {http://nbn-resolving.de/urn:nbn:de:bvb:898-opus4-80606}, pages = {5}, abstract = {Advanced silicon electrode arrays facilitate the recording of thousands of neurons, generating extensive neural data that imposes a significant burden on researchers and processing algorithms. Thus, real-time analysis pipelines are gaining increasing value, while at the same time having to deal with non-stationary and noisy data. We intend to apply Machine Learning (ML) algorithms to a dense set of recordings from rat brains in order to prepare a functional atlas, correlating neuronal signals with anatomical position. While doing so, we needed to decide on a rational way which recording snippet length would best represent the original, longer source sequence and thus suffices to be further processed for anatomical correlation. We implemented an algorithm to evaluate the spectral information of systematically length varied records based on similarity to the original record. For our dataset a recording duration of 3 seconds satisfied moderate requirements across all channels, thus allowing us to reduce computational load for ongoing ML classification of microprobe sourced electrophysiologic signals.}, language = {en} } @incollection{MoralesBaumannPerezetal., author = {Morales, Juan A. and Baumann, Timo and P{\´e}rez, Jos{\´e} L. and Peinado, Antonio M. and G{\´o}mez, {\´A}ngel M.}, title = {Implementaci{\`o}n de un reconocedor distribuido de voz en tiempo real sobre IP}, series = {Actas de las IV Jornadas en Tecnolog{\´i}as del Habla}, booktitle = {Actas de las IV Jornadas en Tecnolog{\´i}as del Habla}, abstract = {En este trabajo se presenta una implementaci{\´o}n de unsistema de reconocimiento distribuido del habla en tiem-po real para su aplicaci{\´o}n en un entorno de Internet. Desa-rrollado como una aplicaci{\´o}n cliente-servidor, el clien-te hace uso del front-end est{\´a}ndar definido por la ETSI.Incluye un detector de voz para s{\´o}lo enviar informaci{\´o}ncuando el locutor habla y la transmisi{\´o}n de la informaci{\´o}nse hace de acuerdo al RFC 3557. El servidor realiza el re-conocimiento utilizando programaci{\´o}n din{\´a}mica, incluyea su vez un detector de voz y utiliza la t{\´e}cnica de mitiga-ci{\´o}n de p{\´e}rdida de paquetes propuesta en el est{\´a}ndar de laETSI. La modularidad del dise{\~n}o permite la utilizaci{\´o}n decualquier otro reconocedor sin que por ello se vean afec-tados los clientes. Las pruebas reales de funcionamientopara la aplicaci{\´o}n particular desarrollada han demostradouna alta fiabilidad en varias condiciones de transmisi{\´o}n.}, language = {es} } @article{NigelGWardVegaBaumann, author = {Nigel G. Ward, and Vega, Alejandro and Baumann, Timo}, title = {Prosodic and Temporal Features for Language Modeling for Dialog}, series = {Speech Communication}, volume = {54}, journal = {Speech Communication}, number = {2}, publisher = {ELSEVIER}, doi = {10.1016/j.specom.2011.07.009}, pages = {161 -- 174}, abstract = {If we can model the cognitive and communicative processes underlying speech, we should be able to better predict what a speaker will do. With this idea as inspiration, we examine a number of prosodic and timing features as potential sources of information on what words the speaker is likely to say next. In spontaneous dialog we find that word probabilities do vary with such features. Using perplexity as the metric, the most informative of these included recent speaking rate, volume, and pitch, and time until end of utterance. Using simple combinations of such features to augment trigram language models gave up to a 8.4\% perplexity benefit on the Switchboard corpus, and up to a 1.0\% relative reduction in word error rate (0.3\% absolute) on the Verbmobil II corpus.}, language = {en} }