@misc{Winkelmann2016, author = {Winkelmann, Stefanie}, title = {Markov Control with Rare State Observation: Average Optimality}, issn = {1438-0064}, url = {http://nbn-resolving.de/urn:nbn:de:0297-zib-60981}, year = {2016}, abstract = {This paper investigates the criterion of long-term average costs for a Markov decision process (MDP) which is not permanently observable. Each observation of the process produces a fixed amount of \textit{information costs} which enter the considered performance criterion and preclude from arbitrarily frequent state testing. Choosing the \textit{rare} observation times is part of the control procedure. In contrast to the theory of partially observable Markov decision processes, we consider an arbitrary continuous-time Markov process on a finite state space without further restrictions on the dynamics or the type of interaction. Based on the original Markov control theory, we redefine the control model and the average cost criterion for the setting of information costs. We analyze the constant of average costs for the case of ergodic dynamics and present an optimality equation which characterizes the optimal choice of control actions and observation times. For this purpose, we construct an equivalent freely observable MDP and translate the well-known results from the original theory to the new setting.}, language = {en} } @misc{WinkelmannSchuette2016, author = {Winkelmann, Stefanie and Sch{\"u}tte, Christof}, title = {The spatiotemporal master equation: approximation of reaction-diffusion dynamics via Markov state modeling}, issn = {1438-0064}, url = {http://nbn-resolving.de/urn:nbn:de:0297-zib-60999}, year = {2016}, abstract = {Accurate modeling and numerical simulation of reaction kinetics is a topic of steady interest. We consider the spatiotemporal chemical master equation (ST-CME) as a model for stochastic reaction-diffusion systems that exhibit properties of metastability. The space of motion is decomposed into metastable compartments and diffusive motion is approximated by jumps between these compartments. Treating these jumps as first-order reactions, simulation of the resulting stochastic system is possible by the Gillespie method. We present the theory of Markov state models (MSM) as a theoretical foundation of this intuitive approach. By means of Markov state modeling, both the number and shape of compartments and the transition rates between them can be determined. We consider the ST-CME for two reaction-diffusion systems and compare it to more detailed models. Moreover, a rigorous formal justification of the ST-CME by Galerkin projection methods is presented.}, language = {en} } @article{WinkelmannSchuette2016, author = {Winkelmann, Stefanie and Sch{\"u}tte, Christof}, title = {The Spatiotemporal Master Equation: Approximation of Reaction-Diffusion Dynamics via Markov State Modeling}, volume = {145}, journal = {Journal of Chemical Physics}, number = {21}, doi = {10.1063/1.4971163}, year = {2016}, abstract = {Accurate modeling and numerical simulation of reaction kinetics is a topic of steady interest.We consider the spatiotemporal chemical master equation (ST-CME) as a model for stochastic reaction-diffusion systems that exhibit properties of metastability. The space of motion is decomposed into metastable compartments and diffusive motion is approximated by jumps between these compartments. Treating these jumps as first-order reactions, simulation of the resulting stochastic system is possible by the Gillespie method. We present the theory of Markov state models (MSM) as a theoretical foundation of this intuitive approach. By means of Markov state modeling, both the number and shape of compartments and the transition rates between them can be determined. We consider the ST-CME for two reaction-diffusion systems and compare it to more detailed models. Moreover, a rigorous formal justification of the ST-CME by Galerkin projection methods is presented.}, language = {en} } @article{KlusSchuette2016, author = {Klus, Stefan and Sch{\"u}tte, Christof}, title = {Towards tensor-based methods for the numerical approximation of the Perron-Frobenius and Koopman operator}, volume = {3}, journal = {Journal of Computational Dynamics}, number = {2}, doi = {10.3934/jcd.2016007}, pages = {139 -- 161}, year = {2016}, abstract = {The global behavior of dynamical systems can be studied by analyzing the eigenvalues and corresponding eigenfunctions of linear operators associated with the system. Two important operators which are frequently used to gain insight into the system's behavior are the Perron-Frobenius operator and the Koopman operator. Due to the curse of dimensionality, computing the eigenfunctions of high-dimensional systems is in general infeasible. We will propose a tensor-based reformulation of two numerical methods for computing finite-dimensional approximations of the aforementioned infinite-dimensional operators, namely Ulam's method and Extended Dynamic Mode Decomposition (EDMD). The aim of the tensor formulation is to approximate the eigenfunctions by low-rank tensors, potentially resulting in a significant reduction of the time and memory required to solve the resulting eigenvalue problems, provided that such a low-rank tensor decomposition exists. Typically, not all variables of a high-dimensional dynamical system contribute equally to the system's behavior, often the dynamics can be decomposed into slow and fast processes, which is also reflected in the eigenfunctions. Thus, the weak coupling between different variables might be approximated by low-rank tensor cores. We will illustrate the efficiency of the tensor-based formulation of Ulam's method and EDMD using simple stochastic differential equations.}, language = {en} }