@article{Winkelmann2017, author = {Winkelmann, Stefanie}, title = {Markov Control with Rare State Observation: Average Optimality}, volume = {23}, journal = {Markov Processes and Related Fields}, pages = {1 -- 34}, year = {2017}, abstract = {This paper investigates the criterion of long-term average costs for a Markov decision process (MDP) which is not permanently observable. Each observation of the process produces a fixed amount of information costs which enter the considered performance criterion and preclude from arbitrarily frequent state testing. Choosing the rare observation times is part of the control procedure. In contrast to the theory of partially observable Markov decision processes, we consider an arbitrary continuous-time Markov process on a finite state space without further restrictions on the dynamics or the type of interaction. Based on the original Markov control theory, we redefine the control model and the average cost criterion for the setting of information costs. We analyze the constant of average costs for the case of ergodic dynamics and present an optimality equation which characterizes the optimal choice of control actions and observation times. For this purpose, we construct an equivalent freely observable MDP and translate the well-known results from the original theory to the new setting.}, language = {en} } @article{WinkelmannSchuette2016, author = {Winkelmann, Stefanie and Sch{\"u}tte, Christof}, title = {The Spatiotemporal Master Equation: Approximation of Reaction-Diffusion Dynamics via Markov State Modeling}, volume = {145}, journal = {Journal of Chemical Physics}, number = {21}, doi = {10.1063/1.4971163}, year = {2016}, abstract = {Accurate modeling and numerical simulation of reaction kinetics is a topic of steady interest.We consider the spatiotemporal chemical master equation (ST-CME) as a model for stochastic reaction-diffusion systems that exhibit properties of metastability. The space of motion is decomposed into metastable compartments and diffusive motion is approximated by jumps between these compartments. Treating these jumps as first-order reactions, simulation of the resulting stochastic system is possible by the Gillespie method. We present the theory of Markov state models (MSM) as a theoretical foundation of this intuitive approach. By means of Markov state modeling, both the number and shape of compartments and the transition rates between them can be determined. We consider the ST-CME for two reaction-diffusion systems and compare it to more detailed models. Moreover, a rigorous formal justification of the ST-CME by Galerkin projection methods is presented.}, language = {en} } @article{DuwalWinkelmannSchuetteetal.2015, author = {Duwal, Sulav and Winkelmann, Stefanie and Sch{\"u}tte, Christof and von Kleist, Max}, title = {Optimal Treatment Strategies in the Context of 'Treatment for Prevention' against HIV/1 in Resource-Poor Settings}, volume = {11}, journal = {PloS Computational Biology}, number = {4}, doi = {10.1371/journal.pcbi.1004200}, year = {2015}, abstract = {An estimated 2.7 million new HIV-1 infections occurred in 2010. `Treatment-for-prevention' may strongly prevent HIV-1 transmission. The basic idea is that immediate treatment initiation rapidly decreases virus burden, which reduces the number of transmittable viruses and thereby the probability of infection. However, HIV inevitably develops drug resistance, which leads to virus rebound and nullifies the effect of `treatment-for-prevention' for the time it remains unrecognized. While timely conducted treatment changes may avert periods of viral rebound, necessary treatment options and diagnostics may be lacking in resource-constrained settings. Within this work, we provide a mathematical platform for comparing different treatment paradigms that can be applied to many medical phenomena. We use this platform to optimize two distinct approaches for the treatment of HIV-1: (i) a diagnostic-guided treatment strategy, based on infrequent and patient-specific diagnostic schedules and (ii) a pro-active strategy that allows treatment adaptation prior to diagnostic ascertainment. Both strategies are compared to current clinical protocols (standard of care and the HPTN052 protocol) in terms of patient health, economic means and reduction in HIV-1 onward transmission exemplarily for South Africa. All therapeutic strategies are assessed using a coarse-grained stochastic model of within-host HIV dynamics and pseudo-codes for solving the respective optimal control problems are provided. Our mathematical model suggests that both optimal strategies (i)-(ii) perform better than the current clinical protocols and no treatment in terms of economic means, life prolongation and reduction of HIV-transmission. The optimal diagnostic-guided strategy suggests rare diagnostics and performs similar to the optimal pro-active strategy. Our results suggest that 'treatment-for-prevention' may be further improved using either of the two analyzed treatment paradigms.}, language = {en} } @article{SchuetteWinkelmannHartmann2012, author = {Sch{\"u}tte, Christof and Winkelmann, Stefanie and Hartmann, Carsten}, title = {Optimal control of molecular dynamics using Markov state models}, volume = {134}, journal = {Math. Program. (Series B)}, number = {1}, doi = {10.1007/s10107-012-0547-6}, pages = {259 -- 282}, year = {2012}, language = {en} } @article{WinkelmannSchuettevonKleist2012, author = {Winkelmann, Stefanie and Sch{\"u}tte, Christof and von Kleist, Max}, title = {Markov Control Processes with Rare State Observation}, volume = {12}, journal = {Communications in Mathematical Sciences}, number = {859}, year = {2012}, language = {en} } @article{WinkelmannSchuettevonKleist2012, author = {Winkelmann, Stefanie and Sch{\"u}tte, Christof and von Kleist, Max}, title = {Markov Control with Rare State Observation}, journal = {International Journal of Biomathematics and Biostatistics}, year = {2012}, language = {en} } @article{DjurdjevacConradHelfmannZonkeretal.2018, author = {Djurdjevac Conrad, Natasa and Helfmann, Luzie and Zonker, Johannes and Winkelmann, Stefanie and Sch{\"u}tte, Christof}, title = {Human mobility and innovation spreading in ancient times: a stochastic agent-based simulation approach}, volume = {7}, journal = {EPJ Data Science}, number = {1}, edition = {EPJ Data Science}, publisher = {EPJ Data Science}, doi = {10.1140/epjds/s13688-018-0153-9}, pages = {24}, year = {2018}, abstract = {Human mobility always had a great influence on the spreading of cultural, social and technological ideas. Developing realistic models that allow for a better understanding, prediction and control of such coupled processes has gained a lot of attention in recent years. However, the modeling of spreading processes that happened in ancient times faces the additional challenge that available knowledge and data is often limited and sparse. In this paper, we present a new agent-based model for the spreading of innovations in the ancient world that is governed by human movements. Our model considers the diffusion of innovations on a spatial network that is changing in time, as the agents are changing their positions. Additionally, we propose a novel stochastic simulation approach to produce spatio-temporal realizations of the spreading process that are instructive for studying its dynamical properties and exploring how different influences affect its speed and spatial evolution.}, language = {en} } @article{NiemannWinkelmannWolfetal.2021, author = {Niemann, Jan-Hendrik and Winkelmann, Stefanie and Wolf, Sarah and Sch{\"u}tte, Christof}, title = {Agent-based modeling: Population limits and large timescales}, volume = {31}, journal = {Chaos: An Interdisciplinary Journal of Nonlinear Science}, number = {3}, issn = {1438-0064}, doi = {10.1063/5.0031373}, url = {http://nbn-resolving.de/urn:nbn:de:0297-zib-77309}, year = {2021}, abstract = {Modeling, simulation and analysis of interacting agent systems is a broad field of research, with existing approaches reaching from informal descriptions of interaction dynamics to more formal, mathematical models. In this paper, we study agent-based models (ABMs) given as continuous-time stochastic processes and their pathwise approximation by ordinary and stochastic differential equations (ODEs and SDEs, respectively) for medium to large populations. By means of an appropriately adapted transfer operator approach we study the behavior of the ABM process on long time scales. We show that, under certain conditions, the transfer operator approach allows to bridge the gap between the pathwise results for large populations on finite timescales, i.e., the SDE limit model, and approaches built to study dynamical behavior on long time scales like large deviation theory. The latter provides a rigorous analysis of rare events including the associated asymptotic rates on timescales that scale exponentially with the population size. We demonstrate that it is possible to reveal metastable structures and timescales of rare events of the ABM process by finite-length trajectories of the SDE process for large enough populations. This approach has the potential to drastically reduce computational effort for the analysis of ABMs.}, language = {en} } @article{HelfmannDjurdjevacConradDjurdjevacetal.2021, author = {Helfmann, Luzie and Djurdjevac Conrad, Natasa and Djurdjevac, Ana and Winkelmann, Stefanie and Sch{\"u}tte, Christof}, title = {From interacting agents to density-based modeling with stochastic PDEs}, volume = {16}, journal = {Communications in Applied Mathematics and Computational Science}, number = {1}, arxiv = {http://arxiv.org/abs/1905.13525}, doi = {10.2140/camcos.2021.16.1}, pages = {1 -- 32}, year = {2021}, abstract = {Many real-world processes can naturally be modeled as systems of interacting agents. However, the long-term simulation of such agent-based models is often intractable when the system becomes too large. In this paper, starting from a stochastic spatio-temporal agent-based model (ABM), we present a reduced model in terms of stochastic PDEs that describes the evolution of agent number densities for large populations. We discuss the algorithmic details of both approaches; regarding the SPDE model, we apply Finite Element discretization in space which not only ensures efficient simulation but also serves as a regularization of the SPDE. Illustrative examples for the spreading of an innovation among agents are given and used for comparing ABM and SPDE models.}, language = {en} } @article{WehlitzPavliotisSchuetteetal.2026, author = {Wehlitz, Nathalie and Pavliotis, Grigorios and Sch{\"u}tte, Christof and Winkelmann, Stefanie}, title = {Data-driven Reduction of Transfer Operators for Particle Clustering Dynamics}, arxiv = {http://arxiv.org/abs/2601.02932}, year = {2026}, abstract = {We develop an operator-based framework to coarse-grain interacting particle systems that exhibit clustering dynamics. Starting from the particle-based transfer operator, we first construct a sequence of reduced representations: the operator is projected onto concentrations and then further reduced by representing the concentration dynamics on a geometric low-dimensional manifold and an adapted finite-state discretization. The resulting coarse-grained transfer operator is finally estimated from dynamical simulation data by inferring the transition probabilities between the Markov states. Applied to systems with multichromatic and Morse interaction potentials, the reduced model reproduces key features of the clustering process, including transitions between cluster configurations and the emergence of metastable states. Spectral analysis and transition-path analysis of the estimated operator reveal implied time scales and dominant transition pathways, providing an interpretable and efficient description of particle-clustering dynamics.}, language = {en} }