@inproceedings{MildeGeislingerLindtetal., author = {Milde, Benjamin and Geislinger, Robert and Lindt, Irina and Baumann, Timo}, title = {Open Source Automatic Lecture Subtitling}, series = {Proceedings of Elektronische Sprachsignalverarbeitung (ESSV): Tagungsband der 32. Konferenz, Berlin, 3.-5. M{\"a}rz 2021}, booktitle = {Proceedings of Elektronische Sprachsignalverarbeitung (ESSV): Tagungsband der 32. Konferenz, Berlin, 3.-5. M{\"a}rz 2021}, publisher = {F{\"o}rderverein Elektronische Sprachsignalverabeitung e.V.}, address = {Berlin, Germany}, url = {http://nbn-resolving.de/urn:nbn:de:gbv:18-228-7-2523}, pages = {128 -- 135}, abstract = {We present a fully automatic solution for German video subtitling, with a focus on lecture videos. We rely entirely on open source models and scripts for German ASR, automatic punctuation reconstruction and subtitle segmentation. All training scripts, 1000h of German speech training data, pre-trained models and the final subtitling program are publicly available. It can readily be integrated into lecture video platforms such as Lecture2Go. The automatically generated subtitles can also serve as a basis to make the video material more accessible (e.g. via search, keyword clouds, and the like) or for further manual revision, potentially helping in significantly speeding up manual work. A particular challenge that we observe in lectures are technical terms that are frequent in a particular lecture, but infrequent in a typical language model and that might be out of vocabulary for a general purpose ASR. We approach this challenge by extracting texts from accompanying lecture slides to adapt the language model of our TDNN-HMM based ASR system. We demonstrate the usability of the full system and its generated subtitles and evaluate on a dataset of manually transcribed lectures with an average of 26.3\% WER.}, language = {en} } @inproceedings{HusseinMeyerSickendiekBaumann, author = {Hussein, Hussein and Meyer-Sickendiek, Burkhard and Baumann, Timo}, title = {How to identify elliptical poems within a digital corpus of auditory poetry}, series = {Elektronische Sprachsignalverarbeitung 2019 (ESSV), Tagungsband der 30. Konferenz, Dresden, 3.-8. M{\"a}rz 2019}, booktitle = {Elektronische Sprachsignalverarbeitung 2019 (ESSV), Tagungsband der 30. Konferenz, Dresden, 3.-8. M{\"a}rz 2019}, editor = {Birkholz, Peter and Stone, Simon}, publisher = {TUDpress}, address = {Dresden, Germany}, isbn = {978-3-959081-57-3}, url = {http://nbn-resolving.de/urn:nbn:de:gbv:18-228-7-2579}, pages = {247 -- 254}, abstract = {Ellipses denote the omission of one or more grammatically necessary phrases. In this paper, we will demonstrate how to identify such ellipses as a rhythmical pattern in modern and postmodern free verse poetry by using data from lyrikline which contain the corresponding audio recording of each poem as spoken by the original author. We present a feature engineering approach based on literary analysis as well as a neural networks based approach for the identification of ellipses within the lines of a poem. A contrast class to the ellipsis is defined from poems consisting of complete and correct sentences. The feature-based approach used features derived from a parser such as verb, comma, and sentence ending punctuation. The classifier of neural networks is trained on the line level to integrate the textual information, the spoken recitation, and the pause information between lines, and to integrate information across the lines within the poem. A statistic analysis of poet's gender showed that 65\% of all elliptical poems were written by female poets. The best results, calculated by the weighted F-measure, for the classification of ellipsis with the contrast class is 0.94 with the neural networks based approach. The best results for classification of elliptical lines is 0.62 with the feature-based approach.}, language = {en} } @inproceedings{BaumannMeyerSickendiekHussein, author = {Baumann, Timo and Meyer-Sickendiek, Burkhard and Hussein, Hussein}, title = {How to Identify Speech When Translating Unpunctuated Poetry}, series = {Proceedings of Elektronische Sprachsignalverarbeitung: Tagungsband der 31. Konferenz, Magdeburg, 4.-6. M{\"a}rz 2020}, booktitle = {Proceedings of Elektronische Sprachsignalverarbeitung: Tagungsband der 31. Konferenz, Magdeburg, 4.-6. M{\"a}rz 2020}, publisher = {F{\"o}rderverein Elektronische Sprachsignalverabeitung e.V.}, address = {Magdeburg, Germany}, url = {http://nbn-resolving.de/urn:nbn:de:gbv:18-228-7-2587}, pages = {165 -- 172}, abstract = {A large proportion of (post)-modern poetry contains no or hardly any punctuation. In our contribution, we will investigate how well punctuation information can be recovered for postmodern poetry based on the information contained in the text and speech of free verse poems. We use the world's largest corpus of spoken (post-)modern poetry from our partner lyrikline which contains the corresponding audio recording of each poem as spoken by the original author and features translations for many of the poems. We identify lines that contain a phrase break in the middle of the poetic line, which may already be helpful for philological analysis on one hand, and identify the position of the break in the line on the other hand. We select those poetic lines that contain one or more punctuation characters that typically indicate a phrase break in poetry (.,;:!?/) somewhere in the middle (rather than only at the end of the line) as our target class. We train a neural network (bidirectional recurrent neural network (RNN) based on gated recurrent units (GRU) with attention) that combines audio and textual features to identify the punctuation with the goal of applying it to reconstruct them within a corpus of unpunctuated poems. Our results clearly indicate that speech is helpful for recovering the constituency structure of post-modern poetry that is partially obfuscated by missing punctuation.}, language = {en} } @inproceedings{BaumannSaboo, author = {Baumann, Timo and Saboo, Ashutosh}, title = {Evaluating Heuristics for Audio-Visual Translation}, series = {Proceedings of the Conference of Computational Humanities Research, (CHR2021): November 17-19, 2021, Amsterdam, The Netherlands}, booktitle = {Proceedings of the Conference of Computational Humanities Research, (CHR2021): November 17-19, 2021, Amsterdam, The Netherlands}, editor = {Ehrmann, Maud}, publisher = {CEUR Workshops}, address = {Amsterdam, the Netherlands}, url = {http://nbn-resolving.de/urn:nbn:de:gbv:18-228-7-2592}, pages = {171 -- 180}, abstract = {Dubbing, i.e., the lip-synchronous translation and revoicing of audio-visual media into a target language from a different source language, is essential for the full-fledged reception of foreign audio-visual media, be it movies, instructional videos or short social media clips. In this paper, we objectify influences on the 'dubbability' of translations, i.e., how well a translation would be synchronously revoiceable to the lips on screen. We explore the value of traditional heuristics used in evaluating the qualitative aspects, in particular matching bilabial consonants and the jaw opening while producing vowels, and control for quantity, i.e., that translations are similar to the source in length. We perform an ablation study using an adversarial neural classifier which is trained to differentiate "true" dubbing translations from machine translations. While we are able to confirm the value of matching lip closure in dubbing, we find that the opening angle of the jaw as determined by the realized vowel may be less relevant than frequently considered in audio-visual translation.}, language = {en} } @inproceedings{Baumann, author = {Baumann, Timo}, title = {Coordinating Speech Delivery to Gesture Progress for Deictic Expressions with Incremental Speech Synthesis}, series = {Proceedings of the Timing in Human-Robot Interaction Workshop held in conjunction with the 9th ACM/IEEE International Conference on Human-Robot Interaction (HRI 2014), Bielefeld, 3. March 2014}, booktitle = {Proceedings of the Timing in Human-Robot Interaction Workshop held in conjunction with the 9th ACM/IEEE International Conference on Human-Robot Interaction (HRI 2014), Bielefeld, 3. March 2014}, url = {http://nbn-resolving.de/urn:nbn:de:gbv:18-228-7-1991}, abstract = {This paper presents ongoing work in incremental speech synthesis that enables a system to adapt speech delivery to unforeseen changes in the timing of motor events (e. g. a robot actuator working faster or slower than anticipated) in order to improve the coordination of speech and gestures for deictic expressions.}, subject = {Sprachverarbeitung}, language = {en} } @misc{MeyerSickendiekBaumannHussein, author = {Meyer-Sickendiek, Burkhard and Baumann, Timo and Hussein, Hussein}, title = {Requirements on the Punctuation Reconstruction for the Translation of Post-modern Poetry}, series = {DHd 2020 ; Spielr{\"a}ume ; Digital Humanities zwischen Modellierung und Interpretation ; Konferenzabstracts ; Universit{\"a}t Paderborn 02. bis 06. M{\"a}rz 2020}, journal = {DHd 2020 ; Spielr{\"a}ume ; Digital Humanities zwischen Modellierung und Interpretation ; Konferenzabstracts ; Universit{\"a}t Paderborn 02. bis 06. M{\"a}rz 2020}, publisher = {Universit{\"a}t Paderborn}, address = {Paderborn}, isbn = {978-3-945437-07-0}, doi = {10.5281/zenodo.4621722}, language = {en} } @misc{Baumann, author = {Baumann, Timo}, title = {Ubiquity of Computing and SLIVAR}, series = {Spoken Language Interaction with Virtual Agents and Robots (SLIVAR): Towards Effective and Ethical Interaction}, volume = {10}, journal = {Spoken Language Interaction with Virtual Agents and Robots (SLIVAR): Towards Effective and Ethical Interaction}, editor = {Devillers, Laurence and Kawahara, Tatsuya and Moore, Roger K. and Scheutz, Matthias}, publisher = {Schloss Dagstuhl - Leibniz-Zentrum f{\"u}r Informatik}, address = {Dagstuhl, Germany}, doi = {10.4230/DagRep.10.1.1}, url = {http://nbn-resolving.de/urn:nbn:de:0030-drops-124009}, pages = {16}, language = {en} } @misc{MeyerSickendiekHusseinBaumann, author = {Meyer-Sickendiek, Burkhard and Hussein, Hussein and Baumann, Timo}, title = {From Fluency To Disfluency: Ranking Prosodic Features Of Poetry By Using Neural Networks}, series = {Proceedings of the International Conference on Digital Humanities (DH 2019), Utrecht, the Netherlands 9-12 July, 2019}, journal = {Proceedings of the International Conference on Digital Humanities (DH 2019), Utrecht, the Netherlands 9-12 July, 2019}, doi = {10.34894/EGZNMI}, language = {en} } @inproceedings{WindbuehlerOkkesimChristetal., author = {Windb{\"u}hler, Anna and Okkesim, S{\"u}kr{\"u} and Christ, Olaf and Mottaghi, Soheil and Rastogi, Shavika and Schmuker, Michael and Baumann, Timo and Hofmann, Ulrich G.}, title = {Machine Learning Approaches to Classify Anatomical Regions in Rodent Brain from High Density Recordings}, series = {44th Annual International Conference of the IEEE Engineering in Medicine and Biology Society (EMBC 2022): 11-15 July 2022, Glasgow, Scotland, United Kingdom}, booktitle = {44th Annual International Conference of the IEEE Engineering in Medicine and Biology Society (EMBC 2022): 11-15 July 2022, Glasgow, Scotland, United Kingdom}, publisher = {IEEE}, doi = {10.1109/EMBC48229.2022.9871702}, url = {http://nbn-resolving.de/urn:nbn:de:bvb:898-opus4-35296}, pages = {3530 -- 3533}, abstract = {Identifying different functional regions during a brain surgery is a challenging task usually performed by highly specialized neurophysiologists. Progress in this field may be used to improve in situ brain navigation and will serve as an important building block to minimize the number of animals in preclinical brain research required by properly positioning implants intraoperatively. The study at hand aims to correlate recorded extracellular signals with the volume of origin by deep learning methods. Our work establishes connections between the position in the brain and recorded high-density neural signals. This was achieved by evaluating the performance of BLSTM, BGRU, QRNN and CNN neural network architectures on multisite electrophysiological data sets. All networks were able to successfully distinguish cortical and thalamic brain regions according to their respective neural signals. The BGRU provides the best results with an accuracy of 88.6 \% and demonstrates that this classification task might be solved in higher detail while minimizing complex preprocessing steps.}, language = {en} } @misc{TsaiBaumannPecuneetal., author = {Tsai, Vivian and Baumann, Timo and Pecune, Florian and Cassell, Justine}, title = {Faster Responses are Better Responses: Introducing Incrementality into Sociable Virtual Personal Assistants}, language = {en} } @misc{PaetzelSchlesingerPricopetal., author = {Paetzel, Maike and Schlesinger, Philipp and Pricop, Mircea and Comaneci, Radu and Baumann, Timo and Menzel, Wolfgang}, title = {Inkrementelle und zustandsbasierte Verarbeitung in einem hybriden Sprachdialogsystem}, language = {en} } @inproceedings{BaumannPaetzelSchlesingeretal., author = {Baumann, Timo and Paetzel, Maike and Schlesinger, Philipp and Menzel, Wolfgang}, title = {Using Affordances to Shape the Interaction in a Hybrid Spoken Dialogue System}, series = {Elektronische Sprachsignalverarbeitung 2013 : Tagungsband der 24. Konferenz, Bielefeld, 26. - 28.3.2013 ; [ESSV 2013]}, booktitle = {Elektronische Sprachsignalverarbeitung 2013 : Tagungsband der 24. Konferenz, Bielefeld, 26. - 28.3.2013 ; [ESSV 2013]}, publisher = {TUDpress}, isbn = {978-3-94431-03-4}, url = {http://nbn-resolving.de/urn:nbn:de:gbv:18-228-7-1836}, pages = {12 -- 19}, abstract = {Affordances manifest possibilities of interaction with a spoken dialogsystem. For example, the act of asking a question affords to the recipient thepossibility of answering. In the system we present, the observable act of maneuveringaffords the possibility of controlling a motion. Our system thus uses the affordanceprinciple to shape the interaction: to trigger the usage of instructions that are easyto understand and process, the system gives immediate visual feedback to afforduser commands that can then be reacted upon. This tightening of the interactionloop requires an incremental processing paradigm to allow fast reactions and to beable to alter ongoing system actions. Our system is a hybrid of incremental andnon-incremental processing components, combining conventional, state graph-basedprocessing, which has the advantage of widely available toolkits and well-understooddialog management, with incremental dialog processing which allows for the tightfeedback loop that provides for quick reactions. We tested our approach in a smalluser study and found that users used simpler and setting-independent commandsmore often and were more efficient when faced with the affordance-based version ofour system.}, language = {en} } @book{Baumann, author = {Baumann, Timo}, title = {Incremental Spoken Dialogue Processing: Architecture and Lower-level Components}, publisher = {Bielefeld University}, isbn = {9783848200221}, url = {http://nbn-resolving.de/urn:nbn:de:hbz:361-25819101}, pages = {176}, language = {en} } @inproceedings{SchulerNayakSahaetal., author = {Schuler, Christian and Nayak, Shravan and Saha, Debjoy and Baumann, Timo}, title = {Can We See Your Response Before You Speak? Exploring Linguistic Information Found in Inter-Utterance Pauses}, series = {Elektronische Sprachsignalverarbeitung 2024, Tagungsband der 35. Konferenz, Regensburg, 6.-8. M{\"a}rz 2024}, booktitle = {Elektronische Sprachsignalverarbeitung 2024, Tagungsband der 35. Konferenz, Regensburg, 6.-8. M{\"a}rz 2024}, editor = {Baumann, Timo}, publisher = {TUDpress}, address = {Dresden}, isbn = {978-3-95908-325-6}, doi = {10.35096/othr/pub-7094}, url = {http://nbn-resolving.de/urn:nbn:de:bvb:898-opus4-70949}, pages = {165 -- 172}, abstract = {In this work we assess whether there is information in pauses in-between utterances of the same or different speakers that are predictive of the following speaker's utterance. We present models that connect a person's visual features before they speak to their upcoming utterance. In our experiments we find that outof-the-box pre-trained models can already reach a better-than-chance performance in correlating video embeddings to utterance embeddings. In contrast, models that attempt to predict the first word after the pause do not outperform a unigram model, indicating that our models do not read lips (based e.g. on co-articulation effects) but rather capture more fundamental aspects of the upcoming utterance.}, language = {en} } @inproceedings{MeyerSickendiekHusseinBaumann, author = {Meyer-Sickendiek, Burkhard and Hussein, Hussein and Baumann, Timo}, title = {Recognizing Modern Sound Poetry with LSTM Networks}, series = {Proceedings of Elektronische Sprachsignalverarbeitung (ESSV)}, booktitle = {Proceedings of Elektronische Sprachsignalverarbeitung (ESSV)}, editor = {Berton, Andr{\´e} and Haiber, Udo and Minker, Wolfgang}, publisher = {TUDpress}, address = {Ulm, Germany}, isbn = {978-3-959081-28-3}, pages = {192 -- 199}, abstract = {Our paper focuses on the computational analysis of "readout poetry" (german: H{\"o}rdichtung) - recordings of poets reading their own work - with regards to the most important type of this genre, the modern "sound poetry" (german: Lautdichtung). Whereas "readout poetry" often uses normal words and sentences, the "sound poetry", developed by dadaistic poets like Hugo Ball and Kurt Schwitters or concrete poets like Ernst Jandl, Oskar Pastior, or Bob Cobbing, combines the "microparticles of the human voice" like the segments in Ernst Jandls sound poem "schtzngrmm" ("schtzngrmm / schtzngrmm / tttt / tttt / grrrmmmmm / tttt / sch / tzngrmm"). Within the genre of sound poetry, there are two main forms: The lettristic and the syllabic decomposition. A short anecdote will explain this difference: The dadaist Raoul Hausmann developed the lettristic sound poetry in his early dadaistic poem "fmsbw" from 1918. This is said to have inspired his successor Schwitters, whose famous "Ursonate" [The Sonata in Primal Speech] begins with the words "F{\"u}mms b{\"o} w{\"o} t{\"a}{\"a} z{\"a}{\"a} Uu". With the "Ursonate", Schwitters developed a syllabic variation of the lettristic poems of Hausmann. The paper shows how to train a bidirectional LSTM network in order to differ between these "dadaistic" sound poems and the "normal" read out poems. In a further step, we will also show how to distinguish between the lettristic and the syllabic decomposition. Based on a bidirectional LSTM network that reads encodings of the character sequence in the poem and uses the output of each directional layer, we identify poems of the sound poetry genre and differentiate between its two types of compositions. The classification results of sound poetry vs. other poetry as well as lettristic vs. syllabic decomposition are with a high performance, yielding a f-scores of 0.86 and 0.84, respectively.}, subject = {Sprachverarbeitung}, language = {en} } @inproceedings{BaumannSiegert, author = {Baumann, Timo and Siegert, Ingo}, title = {Prosodic addressee-detection : ensuring privacy in always-on spoken dialog systems}, series = {Tagungsband Mensch und Computer (MuC'20): 06.09.2020 - 09.09.2020, Magdeburg}, booktitle = {Tagungsband Mensch und Computer (MuC'20): 06.09.2020 - 09.09.2020, Magdeburg}, editor = {Alt, Florian and Schneegass, Stefan and Hornecker, Eva}, publisher = {Association for Computing Machinery}, address = {New York, NY, United States}, isbn = {9781450375405}, doi = {10.1145/3404983.3410021}, pages = {195 -- 198}, abstract = {We analyze the addressee detection task for complexity-identical dialog for both human conversation and device-directed speech. Our recurrent neural model performs at least as good as humans, who have problems with this task, even native speakers, who profit from the relevant linguistic skills. We perform ablation experiments on the features used by our model and show that fundamental frequency variation is the single most relevant feature class. Therefore, we conclude that future systems can detect whether they are addressed based only on speech prosody which does not (or only to a very limited extent) reveal the content of conversations not intended for the system.}, language = {en} } @article{KruegerFischerManoonpongetal., author = {Kr{\"u}ger, Norbert and Fischer, Kerstin and Manoonpong, Poramate and Palinko, Oskar and Bodenhagen, Leon and Baumann, Timo and Kj{\ae}rum, Jens and Rano, Ignacio and Naik, Lakshadeep and Juel, William Kristian and Haarslev, Frederik and Ignasov, Jevgeni and Marchetti, Emanuela and Langedijk, Rosalyn Melissa and Kollakidou, Avgi and Jeppesen, Kasper Camillus and Heidtmann, Conny and Dalgaard, Lars}, title = {The SMOOTH-Robot: A Modular, Interactive Service Robot}, series = {Frontiers in Robotics and AI}, volume = {8}, journal = {Frontiers in Robotics and AI}, publisher = {Frontiers}, doi = {10.3389/frobt.2021.645639}, abstract = {The SMOOTH-robot is a mobile robot that-due to its modularity-combines a relatively low price with the possibility to be used for a large variety of tasks in a wide range of domains. In this article, we demonstrate the potential of the SMOOTH-robot through three use cases, two of which were performed in elderly care homes. The robot is designed so that it can either make itself ready or be quickly changed by staff to perform different tasks. We carefully considered important design parameters such as the appearance, intended and unintended interactions with users, and the technical complexity, in order to achieve high acceptability and a sufficient degree of utilization of the robot. Three demonstrated use cases indicate that such a robot could contribute to an improved work environment, having the potential to free resources of care staff which could be allocated to actual care-giving tasks. Moreover, the SMOOTH-robot can be used in many other domains, as we will also exemplify in this article.}, language = {en} } @article{FischerBodenhagenKruegeretal., author = {Fischer, Kerstin and Bodenhagen, Leon and Kr{\"u}ger, Norbert and Andersen, Morten {\O}stergaard and Baumann, Timo}, title = {Trinvise og delte strategier for kontrol af velf{\ae}rdsrobotter}, series = {Medicoteknik}, volume = {2}, journal = {Medicoteknik}, number = {1}, publisher = {TechMedia}, pages = {11 -- 13}, language = {da} } @inproceedings{KousidisKenningtonBaumannetal., author = {Kousidis, Spyros and Kennington, Casey and Baumann, Timo and Buschmeier, Hendrik and Kopp, Stefan and Schlangen, David}, title = {Situationally Aware In-Car Information Presentation Using Incremental Speech Generation: Safer, and More Effective}, series = {Proceedings of the EACL 2014 Workshop on Dialogue in Motion, Gothenburg, Sweden}, booktitle = {Proceedings of the EACL 2014 Workshop on Dialogue in Motion, Gothenburg, Sweden}, publisher = {Association for Computational Linguistics}, doi = {10.3115/v1/W14-0212}, pages = {68 -- 72}, abstract = {Holding non-co-located conversationswhile driving is dangerous (Horrey and- Wickens, 2006; Strayer et al., 2006), much more so than conversations with physically present, "situated" interlocutors (Drews et al., 2004). In-car dialogue systems typically resemble non-co-located conversations more, and share their negative impact (Strayer et al., 2013). We implemented and tested a simple strategy for making in-car dialogue systems aware of the driving situation, by giving them the capability to interrupt themselves when a dangerous situation is detected,and resume when over. We show that this improves both driving performance and recall of system-presented information, compared to a non-adaptive strategy.}, language = {en} } @inproceedings{NayakSchulerSahaetal., author = {Nayak, Shravan and Schuler, Christian and Saha, Debjoy and Baumann, Timo}, title = {A Deep Dive Into Neural Synchrony Evaluation for Audio-visual Translation}, series = {ICMI '22, Proceedings of the 2022 International Conference on Multimodal Interaction: November 7 - 11 2022, Bengaluru India}, booktitle = {ICMI '22, Proceedings of the 2022 International Conference on Multimodal Interaction: November 7 - 11 2022, Bengaluru India}, editor = {Tumuluri, Raj and Sebe, Nicu and Pingali, Gopal}, publisher = {Association for Computing Machinery}, address = {New York}, isbn = {978-1-4503-9390-4}, doi = {10.1145/3536221.3556621}, pages = {642 -- 647}, abstract = {We present a comprehensive analysis of the neural audio-visual synchrony evaluation tool SyncNet. We assess the agreement of SyncNet scores vis-a-vis human perception and whether we can use these as a reliable metric for evaluating audio-visual lip-synchrony in generation tasks with no ground truth reference audio-video pair. We further look into the underlying elements in audio and video which vitally affect synchrony using interpretable explanations from SyncNet predictions and analyse its susceptibility by introducing adversarial noise. SyncNet has been used in numerous papers on visually-grounded text-to-speech for scenarios such as dubbing. We focus on this scenario which features many local asynchronies (something that SyncNet isn't made for).}, language = {en} } @inproceedings{MelzerDuennweberBaumann, author = {Melzer, Matthias and D{\"u}nnweber, Jan and Baumann, Timo}, title = {Towards Smart Home Data Interpretation Using Analogies to Natural Language Processing}, series = {2022 IEEE International Conference on Smart Internet of Things: IEEE SmartIoT 2022, proceedings, 19-21 August 2022, Suzhou, China, hybrid conference (onsite and virtual)}, booktitle = {2022 IEEE International Conference on Smart Internet of Things: IEEE SmartIoT 2022, proceedings, 19-21 August 2022, Suzhou, China, hybrid conference (onsite and virtual)}, publisher = {IEEE}, isbn = {978-1-6654-7952-3}, doi = {10.1109/SmartIoT55134.2022.00020}, pages = {65 -- 71}, abstract = {Recent advances in the development of smart homes have led to the availability of a wide variety of devices providing a high level of convenience via gesture and speech control or fully automated operation. Many smart home appliances also address the aspects of safety and electricity savings by automatically powering themselves off after not being used for a while. However, many devices remain in a typical household that are not themselves "smart", or are not primarily electric (such as heating systems). We address the savings aspect by identifying processes involving the use of multiple devices in the electrical flow data, as captured by a smart meter in a modern household, rather than focusing on a single appliance. Therefore, we introduce a novel approach to usage pattern analysis based on the idea that a pattern of device usages as a result of a resident's 'routine' (such as making breakfast) can be interpreted similarly to a natural language 'sentence'; Natural Language Processing (NLP) algorithms can then be used for interpreting the residents' behavior. We introduce the notion of bag-of-devices (BoD), derived from the bag-of-words model used in document classification. In an experiment, we show how we use this model to infer predictions about the inhabitants from device usage, such as the resident leaving for the day or just to fetch the newspaper.}, language = {en} } @inproceedings{HusseinMeyerSickendiekBaumann, author = {Hussein, Hussein and Meyer-Sickendiek, Burkhard and Baumann, Timo}, title = {Automatic Detection of Enjambment in German Readout Poetry}, series = {Proceedings of Speech Prosody, 2018, Pozn{\´a}n}, booktitle = {Proceedings of Speech Prosody, 2018, Pozn{\´a}n}, publisher = {ISCA}, doi = {10.21437/SpeechProsody.2018-67}, pages = {329 -- 333}, abstract = {One of the most important patterns in ancient as well as modern poetry is the enjambment, the continuation of a sentence beyond the end of a line, couplet, or stanza. The paper reports first activities towards the development of a digital tool to analyze the accentuation of poetic enjambments in readout poetry. The aim in this contribution is to recognize two forms of enjambment (emphasized and unemphasized) in poems using audio and text data. We use data from lyrikline which is a major online portal for spoken poetry whereas poems are read aloud by the original authors. We identified by hermeneutical means based on literary analysis a total of 69 poems being characteristic for the use of enjambments in modern and postmodern German poetry and train classifiers to differentiate the emphasized/unemphasized ategorization. A remarkable result of our automated analyses (and to our knowledge the first data-driven analysis of this kind) is the identification of a cultural difference in the accentuation of enjambments: statistically speaking, poets from the former GDR tend to emphasize the enjambment, whereas poets from the FRG do not. We use features derived from speech-to-text alignment and statistical parsing information such as pause lengths, number of lines with verbs, and number of lines with punctuation. The best classification results, calculated by the F-measure, for the both types of enjambment (emphasized/unemphasized) is 0.69.}, language = {en} } @inproceedings{BaumannHusseinMeyerSickendiek, author = {Baumann, Timo and Hussein, Hussein and Meyer-Sickendiek, Burkhard}, title = {Analysing the Focus of a Hierarchical Attention Network: the Importance of Enjambments When Classifying Post-modern Poetry}, series = {Proceedings of Interspeech, 2018, Hyderabad}, booktitle = {Proceedings of Interspeech, 2018, Hyderabad}, doi = {10.21437/Interspeech.2018-2533}, pages = {2162 -- 2166}, abstract = {After overcoming the traditional metrics, modern and postmodern poetry developed a large variety of 'free verse prosodies' that falls along a spectrum from a more fluent to a more disfluent and choppy style. We present a method, grounded in philological analysis and theories on cognitive (dis)fluency, to analyze this 'free verse spectrum' into six classes of poetic styles as well as to differentiate three types of poems with enjambments. We use a model for automatic prosodic analysis of spoken free verse poetry which uses deep hierarchical attention networks to integrate the source text and audio and predict the assigned class. We then analyze and fine-tune the model with a particular focus on enjambments and in two ways: we drill down on classification performance by analyzing whether the model focuses on similar traits of poems as humans would, specifically, whether it internally builds a notion of enjambment. We find that our model is similarly good as humans in finding enjambments; however, when we employ the model for classifying enjambment-dominated poem types, it does not pay particular attention to those lines. Adding enjambment labels to the training only marginally improves performance, indicating that all other lines are similarly informative for the model.}, language = {en} } @inproceedings{MeyerSickendiekHusseinBaumann, author = {Meyer-Sickendiek, Burkhard and Hussein, Hussein and Baumann, Timo}, title = {Analysis and Classification of Prosodic Styles in Post-modern Spoken Poetry}, series = {INF-DH 2018 Workshopband}, booktitle = {INF-DH 2018 Workshopband}, editor = {Burghardt, M. and M{\"u}ller-Birn, C.}, publisher = {Gesellschaft f{\"u}r Informatik e.V}, doi = {10.18420/infdh2018-09}, abstract = {We present our research on computer-supported analysis of prosodic styles in post-modern poetry. Our project is unique in making use of both the written as well as the spoken form of the poem as read by the original author. In particular, we use speech and natural language processing technology to align speech and text and to perform textual analyses. We then explore, based on literary theory, the quantitative value of various types of features in differentiating various prosodic classes of post-modern poetry using machine-learning techniques. We contrast this feature-driven approach with a theoretically less informed neural networks-based approach and explore the relative strengths of both models, as well as how to integrate higher-level knowledge into the NN. In this paper, we give an overview of our project, our approach, and particularly focus on the challenges encountered and lessons learned in our interdisciplinary endeavour. The classification results of the rhythmical patterns (six classes) using NN-based approaches are better than by feature-based approaches.}, language = {en} } @inproceedings{BaumannHusseinMeyerSickendiek, author = {Baumann, Timo and Hussein, Hussein and Meyer-Sickendiek, Burkhard}, title = {Analysis of Rhythmic Phrasing: Feature Engineering vs. Representation Learning for Classifying Readout Poetry}, series = {Proceedings of the Second Joint SIGHUM Workshop on Computational Linguistics for Cultural Heritage, Social Sciences, Humanities and Literature (LaTeCH-CLfL-2018), August 25, 2018, Santa Fe, New Mexico, USA}, booktitle = {Proceedings of the Second Joint SIGHUM Workshop on Computational Linguistics for Cultural Heritage, Social Sciences, Humanities and Literature (LaTeCH-CLfL-2018), August 25, 2018, Santa Fe, New Mexico, USA}, publisher = {Association for Computational Linguistics}, address = {Santa Fe, New Mexico}, pages = {44 -- 49}, abstract = {We show how to classify the phrasing of readout poems with the help of machine learning algorithms that use manually engineered features or automatically learn representations. We investigate modern and postmodern poems from the webpage lyrikline, and focus on two exemplary rhythmical patterns in order to detect the rhythmic phrasing: The Parlando and the Variable Foot. These rhythmical patterns have been compared by using two important theoretical works: The Generative Theory of Tonal Music and the Rhythmic Phrasing in English Verse. Using both, we focus on a combination of four different features: The grouping structure, the metrical structure, the time-span-variation, and the prolongation in order to detect the rhythmic phrasing in the two rhythmical types. We use manually engineered features based on text-speech alignment and parsing for classification. We also train a neural network to learn its own representation based on text, speech and audio during pauses. The neural network outperforms manual feature engineering, reaching an f-measure of 0.85.}, language = {en} } @inproceedings{BaumannHusseinMeyerSickendiek, author = {Baumann, Timo and Hussein, Hussein and Meyer-Sickendiek, Burkhard}, title = {Style Detection for Free Verse Poetry from Text and Speech}, series = {Proceedings of the 27th International Conference on Computational Linguistics (COLING), 20-26.08.2018, Santa Fe, New Mexico, USA}, booktitle = {Proceedings of the 27th International Conference on Computational Linguistics (COLING), 20-26.08.2018, Santa Fe, New Mexico, USA}, pages = {1929 -- 1940}, abstract = {Modern and post-modern free verse poems feature a large and complex variety in their poetic prosodies that falls along a continuum from a more fluent to a more disfluent and choppy style. As the poets of modernism overcame rhyme and meter, they oriented themselves in these two opposing directions, creating a free verse spectrum that calls for new analyses of prosodic forms. We present a method, grounded in philological analysis and current research on cognitive (dis)fluency, for automatically analyzing this spectrum. We define and relate six classes of poetic styles (ranging from parlando to lettristic decomposition) by their gradual differentiation. Based on this discussion, we present a model for automatic prosodic classification of spoken free verse poetry that uses deep hierarchical attention networks to integrate the source text and audio and predict the assigned class. We evaluate our model on a large corpus of German author-read post-modern poetry and find that classes can reliably be differentiated, reaching a weighted f-measure of 0.73, when combining textual and phonetic evidence. In our further analyses, we validate the model's decision-making process, the philologically hypothesized continuum of fluency and investigate the relative importance of various features.}, language = {en} } @inproceedings{KoehnBaumannDoerfler, author = {K{\"o}hn, Arne and Baumann, Timo and D{\"o}rfler, Oskar}, title = {An Empirical Analysis of the Correlation of Syntax and Prosody}, series = {Proceedings of Interspeech 2018, 02.-06.09.2018, Hyderabad}, booktitle = {Proceedings of Interspeech 2018, 02.-06.09.2018, Hyderabad}, doi = {10.21437/Interspeech.2018-2530}, pages = {2157 -- 2161}, abstract = {The relation of syntax and prosody (the syntax-prosody interface) has been an active area of research, mostly in linguistics and typically studied under controlled conditions. More recently, prosody has also been successfully used in the data-based training of syntax parsers. However, there is a gap between the controlled and detailed study of the individual effects between syntax and prosody and the large-scale application of prosody in syntactic parsing with only a shallow analysis of the respective influences. In this paper, we close the gap by investigating the significance of correlations of prosodic realization with specific syntactic functions using linear mixed effects models in a very large corpus of read-out German encyclopedic texts. Using this corpus, we are able to analyze prosodic structuring performed by a diverse set of speakers while they try to optimize factual content delivery. After normalization by speaker, we obtain significant effects, e.g. confirming that the subject function, as compared to the object function, has a positive effect on pitch and duration of a word, but a negative effect on loudness.}, language = {en} } @inproceedings{KollerBaumannKoehn, author = {Koller, Alexander and Baumann, Timo and K{\"o}hn, Arne}, title = {DialogOS: Simple and extensible dialog modeling}, series = {Proceedings of Interspeech: Hyderabad, India 2-6 September 2018}, booktitle = {Proceedings of Interspeech: Hyderabad, India 2-6 September 2018}, address = {Hyderabad, India}, abstract = {We present the open-source extensible dialog manager DialogOS. DialogOS features simple finite-state based dialog management (which can be expanded to more complex DM strategies via a full-fledged scripting language) in combination with integrated speech recognition and synthesis in multiple languages. DialogOS runs on all major platforms, provides a simple-to-use graphical interface and can easily be extended via well-defined plugin and client interfaces, or can be integrated server-side into larger existing software infrastructures. We hope that DialogOS will help foster research and teaching given that it lowers the bar of entry into building and testing spoken dialog systems and provides paths to extend one's system as development progresses.}, language = {en} } @inproceedings{Baumann, author = {Baumann, Timo}, title = {Learning to Determine Who is the Better Speaker}, series = {Proceedings of the 9th International Conference on Speech Prosody, 13-16 June 2018, Poznań, Poland}, booktitle = {Proceedings of the 9th International Conference on Speech Prosody, 13-16 June 2018, Poznań, Poland}, publisher = {ISCA}, doi = {10.21437/SpeechProsody.2018-165}, pages = {819 -- 822}, abstract = {Speech can be more or less likable in various ways and comparing speakers by likability has important applications such as speaker selection or matching. Determining the likability of a speaker is a difficult task which can be simplified by breaking it down into pairwise preference decisions. Using a corpus of 5440 pairwise preference ratings collected previously through crowd-sourcing, we train classifiers to determine which of two speakers is "better". We find that modeling the speech feature sequences using LSTMs outperforms conventional methods that pre-aggregate feature averages by a large margin, indicating that the prosodic structure should be taken into account when determining speech quality. Our classifier reaches an accuracy of 97 \% for coarse-grained decisions, where differences between speech quality in both stimuli is relatively large.}, language = {en} } @inproceedings{Baumann, author = {Baumann, Timo}, title = {Highly Responsive Vocal Interaction through Incremental Processing}, series = {Vocal Interactivity in-and-between Humans, Animals and Robots (VIHAR), October 30-4, 2016}, volume = {6}, booktitle = {Vocal Interactivity in-and-between Humans, Animals and Robots (VIHAR), October 30-4, 2016}, number = {10}, editor = {Moore, Roger K. and Thill, Serge and Marxer, Ricard}, publisher = {Schloss Dagstuhl - Leibniz-Zentrum f{\"u}r Informatik}, address = {Dagstuhl, Germany}, doi = {10.4230/DagRep.6.10.154}, pages = {154 -- 194}, abstract = {This seminar was held in late 2016 and brought together, for the first time, researchers studying vocal interaction in a variety of different domains covering communications between all possible combinations of humans, animals, and robots. While each of these sub-domains has extensive histories of research progress, there is much potential for cross-fertilisation that currently remains underexplored. This seminar aimed at bridging this gap. In this report, we present the nascent research field of VIHAR and the major outputs from our seminar in the form of prioritised open research questions, abstracts from stimulus talks given by prominent researchers in their respective fields, and open problem statements by all participants.}, language = {en} } @inproceedings{MeyerSickendiekHusseinBaumann, author = {Meyer-Sickendiek, Burkhard and Hussein, Hussein and Baumann, Timo}, title = {Rhythmicalizer: Data Analysis for the Identification of Rhythmic Patterns in Readout Poetry}, series = {INFORMATIK 2017, Tagung vom 25.- 29. September 2017 in Chemnitz}, booktitle = {INFORMATIK 2017, Tagung vom 25.- 29. September 2017 in Chemnitz}, editor = {Eibl, Maximilian and Gaedke, Martin}, publisher = {Gesellschaft f{\"u}r Informatik}, address = {Bonn}, isbn = {978-3-88579-669-5}, doi = {10.18420/in2017_218}, pages = {2189 -- 2200}, abstract = {The most important development in modern and postmodern poetry is the replacement of traditional meter by new rhythmical patterns. Ever since Walt Whitman's Leaves of Grass (1855), modern (nineteenth-to twenty-first-century) poets have been searching for novel forms of prosody, accent, rhythm, and intonation. Along with the rejection of older metrical units such as the iamb or trochee, a structure of lyrical language was developed that renounced traditional forms like rhyme and meter. This development is subsumed under the term free verse prosody. Our project will test this theory by applying machine learning or deep learning techniques to a corpus of modern and postmodern poems as read aloud by the original authors. To this end, we examine "lyrikline", the most famous online portal for spoken poetry. First, about 17 different patterns being characteristic for the lyrikline-poems have been identified by the philological scholar of this project. This identification was based on a certain philological method including three different steps: a) grammetrical ranking; b) rhythmic phrasing; and c) mapping rubato and prosodic phrasing. In this paper we will show how to combine this philological and a digital analysis by using the prosody detection available in speech processing technology. In order to analyse the data, we want to use different tools for the following tasks: PoS-tagging, alignment, intonation, phrases and pauses, and tempo. We also analyzed the lyrikline-data by identifying the occurrence of the mentioned patterns. This analysis is a first step towards an automatic classification based on machine learning or deep learning techniques.}, language = {en} } @inproceedings{BaumannMeyerSickendiek, author = {Baumann, Timo and Meyer-Sickendiek, Burkhard}, title = {Large-scale Analysis of Spoken Free-verse Poetry}, series = {Proceedings of the Workshop on Language Technology Resources and Tools for Digital Humanities, Osaka, Japan, December 11-17 2016}, booktitle = {Proceedings of the Workshop on Language Technology Resources and Tools for Digital Humanities, Osaka, Japan, December 11-17 2016}, publisher = {The COLING 2016 Organizing Committee}, pages = {125 -- 130}, abstract = {Most modern and post-modern poems have developed a post-metrical idea of lyrical prosody that employs rhythmical features of everyday language and prose instead of a strict adherence to rhyme and metrical schemes. This development is subsumed under the term free verse prosody. We present our methodology for the large-scale analysis of modern and post-modern poetry in both their written form and as spoken aloud by the author. We employ language processing tools to align text and speech, to generate a null-model of how the poem would be spoken by a na{\"i}ve reader, and to extract contrastive prosodic features used by the poet. On these, we intend to build our model of free verse prosody, which will help to understand, differentiate and relate the different styles of free verse poetry. We plan to use our processing scheme on large amounts of data to iteratively build models of styles, to validate and guide manual style annotation, to identify further rhythmical categories, and ultimately to broaden our understanding of free verse poetry. In this paper, we report on a proof-of-concept of our methodology using smaller amounts of poems and a limited set of features. We find that our methodology helps to extract differentiating features in the authors' speech that can be explained by philological insight. Thus, our automatic method helps to guide the literary analysis and this in turn helps to improve our computational models.}, language = {en} } @inproceedings{HusseinMeyerSickendiekBaumann, author = {Hussein, Hussein and Meyer-Sickendiek, Burkhard and Baumann, Timo}, title = {Tonality in Language: The Generative Theory of Tonal Music as a Framework for Prosodic Analysis of Poetry}, series = {6th International Symposium on Tonal Aspects of Languages (TAL 2018), 18-20 June 2018, Berlin, Germany}, booktitle = {6th International Symposium on Tonal Aspects of Languages (TAL 2018), 18-20 June 2018, Berlin, Germany}, publisher = {ISCA}, doi = {10.21437/TAL.2018-36}, pages = {178 -- 182}, abstract = {This contribution focuses on structural similarities between tonality and cadences in music on the one hand, and rhythmical patterns in poetic languages respectively poetry on the other hand. We investigate two exemplary rhythmical patterns in modern and postmodern poetry to detect these tonality-like features in poetic language: The Parlando and the Variable Foot. German poems readout from the original poets are collected from the webpage of our partner lyrikline. We compared these rhythmical features with tonality rules, explained in two important theoretical volumes: The Generative Theory of Tonal Music and the Rhythmic Phrasing in English Verse. Using both volumes, we focused on a certain combination of four different features: The grouping structure, the metrical structure, the time-span-variation and the prolongation, in order to detect the two important rhythmical patterns which use tonality-like features in poetic language (Parlando and Variable Foot). Different features including pause and parser information are used in this classification process. The best classification result, calculated by the f-measure, for Parlando and Variable Foot is 0.69.}, language = {en} } @incollection{BaumannKenningtonHoughetal., author = {Baumann, Timo and Kennington, Casey and Hough, Julian and Schlangen, David}, title = {Recognising Conversational Speech: What an Incremental ASR Should Do for a Dialogue System and How to Get There}, series = {Dialogues with social robots: Enablements analyses and evaluation}, booktitle = {Dialogues with social robots: Enablements analyses and evaluation}, editor = {Jokinen, Kristiina and Wilcock, Graham}, publisher = {Springer}, address = {Singapore}, isbn = {978-981-10-2584-6}, doi = {10.1007/978-981-10-2585-3_35}, pages = {421 -- 432}, abstract = {Automatic speech recognition (ASR) is not only becoming increasingly accurate, but also increasingly adapted for producing timely, incremental output. However, overall accuracy and timeliness alone are insufficient when it comes to interactive dialogue systems which require stability in the output and responsivity to the utterance as it is unfolding. Furthermore, for a dialogue system to deal with phenomena such as disfluencies, to achieve deep understanding of user utterances these should be preserved or marked up for use by downstream components, such as language understanding, rather than be filtered out. Similarly, word timing can be informative for analyzing deictic expressions in a situated environment and should be available for analysis. Here we investigate the overall accuracy and incremental performance of three widely used systems and discuss their suitability for the aforementioned perspectives. From the differing performance along these measures we provide a picture of the requirements for incremental ASR in dialogue systems and describe freely available tools for using and evaluating incremental ASR.}, language = {en} } @inproceedings{KoehnBaumann, author = {K{\"o}hn, Arne and Baumann, Timo}, title = {Predictive Incremental Parsing Helps Language Modeling}, series = {Proceedings of COLING 2016, the 26th International Conference on Computational Linguistics: Technical Papers, Osaka, Japan}, booktitle = {Proceedings of COLING 2016, the 26th International Conference on Computational Linguistics: Technical Papers, Osaka, Japan}, publisher = {The COLING 2016 Organizing Committee}, pages = {268 -- 277}, abstract = {Predictive incremental parsing produces syntactic representations of sentences as they are produced, e.g. by typing or speaking. In order to generate connected parses for such unfinished sentences, upcoming word types can be hypothesized and structurally integrated with already realized words. For example, the presence of a determiner as the last word of a sentence prefix may indicate that a noun will appear somewhere in the completion of that sentence, and the determiner can be attached to the predicted noun. We combine the forward-looking parser predictions with backward-looking N-gram histories and analyze in a set of experiments the impact on language models, i.e. stronger discriminative power but also higher data sparsity. Conditioning N-gram models, MaxEnt models or RNN-LMs on parser predictions yields perplexity reductions of about 6\%. Our method (a) retains online decoding capabilities and (b) incurs relatively little computational overhead which sets it apart from previous approaches that use syntax for language modeling. Our method is particularly attractive for modular systems that make use of a syntax parser anyway, e.g. as part of an understanding pipeline where predictive parsing improves language modeling at no additional cost.}, language = {en} } @inproceedings{Baumann, author = {Baumann, Timo}, title = {Large-Scale Speaker Ranking from Crowdsourced Pairwise Listener Ratings}, series = {Interspeech 2017 : 20-24 August 2017}, booktitle = {Interspeech 2017 : 20-24 August 2017}, publisher = {ISCA}, doi = {10.21437/Interspeech.2017-1697}, pages = {2262 -- 2266}, abstract = {Speech quality and likability is a multi-faceted phenomenon consisting of a combination of perceptory features that cannot easily be computed nor weighed automatically. Yet, it is often easy to decide which of two voices one likes better, even though it would be hard to describe why, or to name the underlying basic perceptory features. Although likability is inherently subjective and individual preferences differ frequently, generalizations are useful and there is often a broad intersubjective consensus about whether one speaker is more likable than another. However, breaking down likability rankings into pairwise comparisons leads to a quadratic explosion of rating pairs. We present a methodology and software to efficiently create a likability ranking for many speakers from crowdsourced pairwise likability ratings. We collected pairwise likability ratings for many (>220) speakers from many raters (>160) and turn these ratings into one likability ranking. We investigate the resulting speaker ranking stability under different conditions: limiting the number of ratings and the dependence on rater and speaker characteristics. We also analyze the ranking wrt. acoustic correlates to find out what factors influence likability. We publish our ranking and the underlying ratings in order to facilitate further research.}, language = {en} } @inproceedings{RohdeBaumann, author = {Rohde, Marcel and Baumann, Timo}, title = {Navigating the Spoken Wikipedia}, series = {Proceedings of the 7th Workshop on Speech and Language Processing for Assistive Technologies (SLPAT 2016)}, booktitle = {Proceedings of the 7th Workshop on Speech and Language Processing for Assistive Technologies (SLPAT 2016)}, doi = {10.21437/SLPAT.2016-2}, pages = {9 -- 13}, abstract = {The Spoken Wikipedia project unites volunteer readers of encyclopedic entries. Their recordings make encyclopedic knowledge accessible to persons who are unable to read (out of alexia, visual impairment, or because their sight is currently occupied, e. g. while driving). However, on Wikipedia, recordings are available as raw audio files that can only be consumed linearly, without the possibility for targeted navigation or search. We present a reading application which uses an alignment between the recording, text and article structure and which allows to navigate spoken articles, through a graphical or voice-based user interface (or a combination thereof). We present the results of a usability study in which we compare the two interaction modalities. We find that both types of interaction enable users to navigate articles and to find specific information much more quickly compared to a sequential presentation of the full article. In particular when the VUI is not restricted by speech recognition and understanding issues, this interface is on par with the graphical interface and thus a real option for browsing the Wikipedia without the need for vision or reading.}, language = {en} } @inproceedings{KoehnStegenBaumann, author = {K{\"o}hn, Arne and Stegen, Florian and Baumann, Timo}, title = {Mining the Spoken Wikipedia for Speech Data and Beyond}, series = {Proceedings of the Tenth International Conference on Language Resources and Evaluation (LREC'16)}, booktitle = {Proceedings of the Tenth International Conference on Language Resources and Evaluation (LREC'16)}, publisher = {European Language Resources Association (ELRA)}, pages = {4644 -- 4647}, abstract = {We present a corpus of time-aligned spoken data of Wikipedia articles as well as the pipeline that allows to generate such corpora for many languages. There are initiatives to create and sustain spoken Wikipedia versions in many languages and hence the data is freely available, grows over time, and can be used for automatic corpus creation. Our pipeline automatically downloads and aligns this data. The resulting German corpus currently totals 293h of audio, of which we align 71h in full sentences and another 86h of sentences with some missing words. The English corpus consists of 287h, for which we align 27h in full sentence and 157h with some missing words. Results are publically available.}, language = {en} } @inproceedings{KirschSteinickeBaumann, author = {Kirsch, Kolja and Steinicke, Frank and Baumann, Timo}, title = {Entwicklung und Evaluierung einer Notenwender-App f{\"u}r Klaviernoten}, series = {Mensch und Computer 2016 - Tagungsband, 4. - 7. September 2016, Aachen}, booktitle = {Mensch und Computer 2016 - Tagungsband, 4. - 7. September 2016, Aachen}, publisher = {Gesellschaft f{\"u}r Informatik e.V.}, address = {Aachen}, doi = {10.18420/muc2016-mci-0159}, abstract = {Das st{\"a}ndige Umbl{\"a}ttern von Noten ist f{\"u}r Musiker ein wiederkehrendes Problem. Dieses wird h{\"a}ufig durch einen Assistenten des Musikers, dem sogenannten Notenwender, gel{\"o}st. Diese Unterst{\"u}tzung haben allerdings viele Musiker nur selten w{\"a}hrend des {\"U}bens. In diesem Artikel stellen wir eine Anwendung f{\"u}r mobile Ger{\"a}te vor, die auf verschiedene Arten das Umbl{\"a}ttern von Klavierpartituren unterst{\"u}tzt. In einer Studie mit professionellen Musikern und Klaviersch{\"u}lern wurden diese Arten gegeneinander abgewogen. Die Ergebnisse zeigen auf, dass computer-unterst{\"u}tztes Bl{\"a}ttern Vorteile gegen{\"u}ber herk{\"o}mmlichem Bl{\"a}ttern hat.}, language = {de} } @inproceedings{TwiefelBaumannHeinrichetal., author = {Twiefel, Johannes and Baumann, Timo and Heinrich, Stefan and Wermter, Stefan}, title = {Improving Domain-independent Cloud-based Speech Recognition with Domain-dependent Phonetic Post-processing}, series = {AAAI 2014 - Twenty-Eighth AAAI Conference on Artificial Intelligence (AAAI-14), Qu{\´e}bec, CA, July 27-31, 2014}, booktitle = {AAAI 2014 - Twenty-Eighth AAAI Conference on Artificial Intelligence (AAAI-14), Qu{\´e}bec, CA, July 27-31, 2014}, publisher = {Association for the Advancement of Artificial Intelligence}, doi = {10.1609/aaai.v28i1.8929}, pages = {1529 -- 1535}, abstract = {Automatic speech recognition (ASR) technology has been developed to such a level that off-the-shelf distributed speech recognition services are available (free of cost), which allow researchers to integrate speech into their applications with little development effort or expert knowledge leading to better results compared with previously used open-source tools. Often, however, such services do not accept language models or grammars but process free speech from any domain. While results are very good given the enormous size of the search space, results frequently contain out-of-domain words or constructs that cannot be understood by subsequent domain-dependent natural language understanding (NLU) components. We present a versatile post-processing technique based on phonetic distance that integrates domain knowledge with open-domain ASR results, leading to improved ASR performance. Notably, our technique is able to make use of domain restrictions using various degrees of domain knowledge, ranging from pure vocabulary restrictions via grammars or N-Grams to restrictions of the acceptable utterances. We present results for a variety of corpora (mainly from human-robot interaction) where our combined approach significantly outperforms Google ASR as well as a plain open-source ASR solution.}, language = {en} } @inproceedings{KenningtonKousidisBaumannetal., author = {Kennington, Casey and Kousidis, Spyros and Baumann, Timo and Buschmeier, Hendrik and Kopp, Stefan and Schlangen, David}, title = {Better Driving and Recall When In-car Information Presentation Uses Situationally-Aware Incremental Speech Output Generation}, series = {Proceedings of the 6th International Conference on Automotive User Interfaces and Interactive Vehicular Applications, AutomotiveUI '14, Seattle WA, USA, September 17 - 19, 2014}, booktitle = {Proceedings of the 6th International Conference on Automotive User Interfaces and Interactive Vehicular Applications, AutomotiveUI '14, Seattle WA, USA, September 17 - 19, 2014}, editor = {Miller, Erika and Wu, Yuqing}, publisher = {ACM}, address = {New York, NY}, doi = {10.1145/2667317.2667332}, pages = {1 -- 7}, abstract = {It is established that driver distraction is the result of sharing cognitive resources between the primary task (driving) and any other secondary task. In the case of holding conversations, a human passenger who is aware of the driving conditions can choose to interrupt his speech in situations potentially requiring more attention from the driver, but in-car information systems typically do not exhibit such sensitivity. We have designed and tested such a system in a driving simulation environment. Unlike other systems, our system delivers information via speech (calendar entries with scheduled meetings) but is able to react to signals from the environment to interrupt when the driver needs to be fully attentive to the driving task and subsequently resume its delivery. Distraction is measured by a secondary short-term memory task. In both tasks, drivers perform significantly worse when the system does not adapt its speech, while they perform equally well to control conditions (no concurrent task) when the system intelligently interrupts and resumes.}, language = {en} } @inproceedings{Baumann, author = {Baumann, Timo}, title = {Partial Representations Improve the Prosody of Incremental Speech Synthesis}, series = {Interspeech 2014, Singapore, 14-18 September 2014}, booktitle = {Interspeech 2014, Singapore, 14-18 September 2014}, doi = {10.21437/Interspeech.2014-199}, pages = {2932 -- 2936}, abstract = {When humans speak, they do not plan their full utterance inall detail before beginning to speak, nor do they speak piece-by-piece and ignoring their full message - instead humans usepartial representations in which they fill in the missing partsas the utterance unfolds. Incremental speech synthesizers, incontrast, have not yet made use of partial representations and theinformation contained there-in.We analyze the quality of prosodic parameter assignments(pitch and duration) generated from partial utterance specifi-cations (substituting defaults for missing features) in order todetermine the requirements that symbolic incremental prosodymodelling should meet. We find that broader, higher-level infor-mation helps to improve prosody even if lower-level informationabout the near future is yet unavailable. Furthermore, we findthat symbolic phrase-level or utterance-level information is mosthelpful towards the end of the phrase or utterance, respectively,that is, when this information is becoming available even in theincremental case. Thus, the negative impact of incremental pro-cessing can be minimized by using partial representations thatare filled in incrementally.}, language = {en} } @inproceedings{KousidisKenningtonBaumannetal., author = {Kousidis, Spyros and Kennington, Casey and Baumann, Timo and Buschmeier, Hendrik and Kopp, Stefan and Schlangen, David}, title = {A Multimodal In-Car Dialogue System That Tracks The Driver's Attention}, series = {Proceedings of the 16th International Conference on Multimodal Interaction, ICMI '14, November 12 - 16 2014, Istanbul}, booktitle = {Proceedings of the 16th International Conference on Multimodal Interaction, ICMI '14, November 12 - 16 2014, Istanbul}, editor = {Ali Salah, Albert}, publisher = {ACM}, address = {New York, NY}, isbn = {9781450328852}, doi = {10.1145/2663204.2663244}, pages = {26 -- 33}, abstract = {When a passenger speaks to a driver, he or she is co-located with the driver, is generally aware of the situation, and can stop speaking to allow the driver to focus on the driving task. In-car dialogue systems ignore these important aspects, making them more distracting than even cell-phone conversations. We developed and tested a "situationally-aware" dialogue system that can interrupt its speech when a situation which requires more attention from the driver is detected, and can resume when driving conditions return to normal. Furthermore, our system allows driver-controlled resumption of interrupted speech via verbal or visual cues (head nods). Over two experiments, we found that the situationally-aware spoken dialogue system improves driving performance and attention to the speech content, while driver-controlled speech resumption does not hinder performance in either of these two tasks}, language = {en} } @inproceedings{BaumannLindner, author = {Baumann, Timo and Lindner, Felix}, title = {Incremental Speech Production for Polite and Natural Personal-Space Intrusion}, series = {Social robotics : 7th International Conference, ICSR 2015, Paris, France, October 26-30, 2015, Proceedings}, booktitle = {Social robotics : 7th International Conference, ICSR 2015, Paris, France, October 26-30, 2015, Proceedings}, editor = {Tapus, Adriana and Andr{\´e}, Elisabeth and Martin, Jean-Claude}, publisher = {Springer}, address = {Cham}, isbn = {978-3-319-25553-8}, doi = {10.1007/978-3-319-25554-5_8}, pages = {72 -- 82}, abstract = {We propose to use a model of personal space to initiate communication while passing a human thereby acknowledging that humans are not just a special kind of obstacle to be avoided but potential interaction partners. As a simple form of interaction, our system communicates an apology while closely passing a human. To this end, we present a software architecture that integrates a social-spaces knowledge base and a component for incremental speech production. Incrementality ensures that the robot's utterance can be adapted to fit the developing situation in a natural way. Observer ratings show that personal-space intrusion is perceived as both natural and polite if the robot has the capability to utter and adapt an apology in an incremental way whereas it is perceived as unfriendly if the robot intrudes personal space without saying anything. Moreover, the robot is perceived as less natural if it does not adapt.}, language = {en} } @inproceedings{BaumannKenningtonHoughetal., author = {Baumann, Timo and Kennington, Casey and Hough, Julian and Schlangen, David}, title = {Recognising Conversational Speech: What an Incremental ASR Should Do for a Dialogue System and How to Get There}, series = {Proceedings of the International Workshop Series on Spoken Dialogue Systems Technology (IWSDS) 2016}, booktitle = {Proceedings of the International Workshop Series on Spoken Dialogue Systems Technology (IWSDS) 2016}, abstract = {Automatic speech recognition (ASR) is not only becoming increasingly accurate, but also increasingly adapted for producing timely, incremental output. However, overall accuracy and timeliness alone are insufficient when it comes to interactive dialogue systems which require stability in the output and responsivity to the utterance as it is unfolding. Furthermore, for a dialogue system to deal with phenomena such as disfluencies, to achieve deep understanding of user utterances these should be preserved or marked up for use by downstream components, such as language understanding, rather than be filtered out. Similarly, word timing can be informative for analyzing deictic expressions in a situated environment and should be available for analysis. Here we investigate the overall accuracy and incremental performance of three widely used systems and discuss their suitability for the aforementioned perspectives. From the differing performance along these measures we provide a picture of the requirements for incremental ASR in dialogue systems and describe freely available tools for using and evaluating incremental ASR.}, language = {en} } @inproceedings{BaumannLindner, author = {Baumann, Timo and Lindner, Felix}, title = {Towards a Social Robot that Incrementally Justifies Personal-Space Intrusion}, series = {Proceedings of the Workshop Towards a Framework for Joint Action 2015}, booktitle = {Proceedings of the Workshop Towards a Framework for Joint Action 2015}, abstract = {Robots should appropriately give reasons for their actions when these actions affect a human's action or goal space. Communicating reasons may help the human understand the robot's intents and may initiate joint action, i. e., accepting the robot's goals and cooperating on the robot's actions. However, to be efficient, the communication of reasons should be limited to the necessary rather than to completeness, conforming to the Gricean Maxim of Quantity. Furthermore, what is necessary only becomes apparent as the situation evolves and hence, for seamless interaction, ongoing utterances must be adapted as they happen. We present a system that flexibly gives reasons in a reduced setting in which the robot needs to intrude a human's personal space in order to reach its goal.}, language = {en} } @inproceedings{PougetHueberBaillyetal., author = {Pouget, Ma{\"e}l and Hueber, Thomas and Bailly, G{\´e}rard and Baumann, Timo}, title = {HMM training strategy for incremental speech synthesis}, series = {Interspeech 2015, Dresden, Germany, 6-10 September 2015}, booktitle = {Interspeech 2015, Dresden, Germany, 6-10 September 2015}, publisher = {ISCA}, doi = {10.21437/Interspeech.2015-304}, pages = {1201 -- 1205}, abstract = {Incremental speech synthesis aims at delivering the synthetic voice while the sentence is still being typed. One of the main challenges is the online estimation of the target prosody from a partial knowledge of the sentence's syntactic structure. In the context of HMM-based speech synthesis, this typically results in missing segmental and suprasegmental features, which describe the linguistic context of each phoneme. This study describes a voice training procedure which integrates explicitly a potential uncertainty on some contextual features. The proposed technique is compared to a baseline approach (previously published), which consists in substituting a missing contextual feature by a default value calculated on the training set. Both techniques were implemented in a HMM-based Text-To-Speech system for French, and compared using objective and perceptual measurements. Experimental results show that the proposed strategy outperforms the baseline technique for this language.}, language = {en} } @inproceedings{Baumann, author = {Baumann, Timo}, title = {Decision Tree Usage for Incremental Parametric Speech Synthesis}, series = {Proceedings of the International Conference on Acoustics, Speech, and Signal Processing (ICASSP) 2014, 4-9 May 2014, Florence, Italy}, booktitle = {Proceedings of the International Conference on Acoustics, Speech, and Signal Processing (ICASSP) 2014, 4-9 May 2014, Florence, Italy}, publisher = {IEEE}, doi = {10.1109/ICASSP.2014.6854316}, pages = {3847 -- 3851}, abstract = {Human speakers plan and deliver their utterances incrementally, piece-by-piece, and it is obvious that their choice regarding phonetic details (and the details' peculiarities) is rarely determined by globally optimal solutions. In contrast, parametric speech synthesizers use a full-utterance context when optimizing vocoding parameters and when determing HMM states. Apart from being cognitively implausible, this impedes incremental use-cases, where the future context is often at least partially unavailable. This paper investigates the `locality' of features in parametric speech synthesis voices and takes some missing steps towards better HMM state selection and prosody modelling for incremental speech synthesis.}, language = {en} } @inproceedings{BaumannBangaloreHirschberg, author = {Baumann, Timo and Bangalore, Srinivas and Hirschberg, Julia}, title = {Towards Simultaneous Interpreting: The Timing of Incremental Machine Translation and Speech Synthesis}, series = {Proceedings of the 11th International Workshop on Spoken Language Translation. Papers. Lake Tahoe, December 4th and 5th, 2014}, booktitle = {Proceedings of the 11th International Workshop on Spoken Language Translation. Papers. Lake Tahoe, December 4th and 5th, 2014}, editor = {Federico, Marcello and St{\"u}ker, Sebastian and Yvon, Fran{\c{c}}ois}, pages = {163 -- 168}, abstract = {In simultaneous interpreting, human experts incrementally construct and extend partial hypotheses about the source speaker's message, and start to verbalize a corresponding message in the target language, based on a partial translation - which may have to be corrected occasionally. They commence the target utterance in the hope that they will be able to finish understanding the source speaker's message and determine its translation in time for the unfolding delivery. Of course, both incremental understanding and translation by humans can be garden-pathed, although experts are able to optimize their delivery so as to balance the goals of minimal latency, translation quality and high speech fluency with few corrections. We investigate the temporal properties of both translation input and output to evaluate the tradeoff between low latency and translation quality. In addition, we estimate the improvements that can be gained with a tempo-elastic speech synthesizer.}, language = {en} } @article{MeyerSickendiekHusseinBaumann, author = {Meyer-Sickendiek, Burkhard and Hussein, Hussein and Baumann, Timo}, title = {Towards the Creation of a Poetry Translation Mapping System}, series = {Archives of Data Science, Series A}, volume = {5}, journal = {Archives of Data Science, Series A}, number = {1}, publisher = {Towards the Creation of a Poetry Translation Mapping System}, issn = {2363-9881}, doi = {10.5445/KSP/1000087327/21}, pages = {1 -- 15}, abstract = {The translation of poetry is a complex, multifaceted challenge: the translated text should communicate the same meaning, similar metaphoric expressions, and also match the style and prosody of the original poem. Research on machine poetry translation is existing since 2010, but for four reasons it is still rather insufficient: 1. The few approaches existing completely lack any knowledge about current developments in both lyric theory and translation theory. 2. They are based on very small datasets. 3. They mostly ignored the neural learning approach that superseded the long-standing dominance of phrase-based approaches within machine translation. 4. They have no concept concerning the pragmatic function of their research and the resulting tools. Our paper describes how to improve the existing research and technology for poetry translations in exactly these four points. With regards to 1) we will describe the "Poetics of Translation". With regards to 2) we will introduce the Worlds largest corpus for poetry translations from lyrikline. With regards to 3) we will describe first steps towards a neural machine translation of poetry. With regards to 4) we will describe first steps towards the development of a poetry translation mapping system.}, language = {en} } @misc{WardBaumannKarkhedkaretal., author = {Ward, Nigel and Baumann, Timo and Karkhedkar, Shreyas and Novick, David}, title = {Dynamic control of voice codec data rate}, abstract = {A method, system, and computer-usable non-transitory storage device for dynamic voice codec adaptation are disclosed. The voice codec adapts in real time to devote more bits to audio quality when it is most needed, and fewer bits to less important parts of utterances are disclosed. Dialog knowledge is utilized for compression opportunities to adjust the bitrate moment-by-moment, based on the inferred value of each frame. Frame importance and appropriate transmission fidelity is predicted based on prosodic features and models of dialog dynamics. This technique provides the same communications quality with less spectrum needs, fewer antennas, and less battery drain.}, language = {en} } @inproceedings{JohannssenBiemannRemusetal., author = {Johannßen, Dirk and Biemann, Chris and Remus, Steffen and Baumann, Timo and Scheffer, David}, title = {GermEval 2020 Task 1 on the Classification and Regression of Cognitive and Motivational style from Text}, series = {Proceedings of the 5th SwissText \& 16th KONVENS Joint Conference 2020}, booktitle = {Proceedings of the 5th SwissText \& 16th KONVENS Joint Conference 2020}, address = {Zurich, Switzerland}, pages = {1 -- 10}, abstract = {This paper describes the tasks, databases, baseline systems, and summarizes submissions and results for the GermEval 2020 Shared Task 1 on the Classification and Regression of Cognitive and Motivational Style from Text. This shared task is divided into two subtasks, a regression task, and a classification task. Subtask 1 asks participants to reproduce a ranking of students based on average aptitude indicators such as different high school grades and different IQ scores. The second subtask aims to classify so-called implicit motives, which are projective testing procedures that can reveal unconscious desires. Besides five implicit motives, the target labels of Subtask 2 also contain one of six levels that describe the type of self-regulation when acting out a motive, which makes this task a multiclass-classification with 30 target labels. 3 participants submitted multiple systems. Subtask 1 was solved (best r = .3701) mainly with non-neural systems and statistical language representations, submissions for Subtask 2 utilized neural approaches and word embeddings (best macro F1 = 70.40). Not only were the tasks solvable, analyses by the participants even showed connections to the implicit psychometrics theory and behavioral observations made by psychologists. This paper describes the tasks, databases, baseline systems, and summarizes submissions and results for the GermEval 2020 Shared Task 1 on the Classification and Regression of Cognitive and Motivational Style from Text. This shared task is divided into two subtasks, a regression task, and a classification task. Subtask 1 asks participants to reproduce a ranking of students based on average aptitude indicators such as different high school grades and different IQ scores. The second subtask aims to classify so-called implicit motives, which are projective testing procedures that can reveal unconscious desires. Besides five implicit motives, the target labels of Subtask 2 also contain one of six levels that describe the type of self-regulation when acting out a motive, which makes this task a multiclass-classification with 30 target labels. 3 participants submitted multiple systems. Subtask 1 was solved (best r =.3701) mainly with non-neural systems and statistical language representations, submissions for Subtask 2 utilized neural approaches and word embeddings (best macro F1 = 70.40). Not only were the tasks solvable, analyses by the participants even showed connections to the implicit psychometrics theory and behavioral observations made by psychologists.}, language = {en} } @inproceedings{HusseinMeyerSickendiekBaumann, author = {Hussein, Hussein and Meyer-Sickendiek, Burkhard and Baumann, Timo}, title = {Free Verse and Beyond: How to Classify Post-modern Spoken Poetry}, series = {Proceedings of Speech Prosody: Tokyo, Japan, 25-28 May 2020}, booktitle = {Proceedings of Speech Prosody: Tokyo, Japan, 25-28 May 2020}, publisher = {ISCA}, doi = {10.21437/SpeechProsody.2020-141}, pages = {690 -- 694}, abstract = {This paper presents the classification of rhythmical patterns detected in post-modern spoken poetry by means of machine learning algorithms that use manually engineered features or automatically learnt representations. We used the world's largest corpus of spoken poetry from our partner lyrikline. We identified nine rhythmical patterns within a spectrum raging from a more fluent to a more disfluent poetic style. The text data analyzed by a statistical parser. Prosodic features of rhythmical patterns are identified by using the parser information. For the classification of rhythmical patterns, we used a neural networks-based approach which use text, audio, and pause information between poetic lines as features. Different combinations of features as well as the integration of feature engineering in the neural networks-based approach are tested. We compared the performance of both approaches (feature-based and neural network-based) using combinations of different features. The results show - by using the weighted average of f-measure for the evaluation - that the neural networks-based approach performed much better in classification of rhythmical patterns. The important improvement of the classification results lies in the use of the audio information. The integration of feature engineering in the neural networks-based approach yielded a very small result improvement.}, language = {en} }