@misc{WinkelmannSchuettevonKleist2013, author = {Winkelmann, Stefanie and Sch{\"u}tte, Christof and von Kleist, Max}, title = {Markov Control Processes with Rare State Observation: Theory and Application to Treatment Scheduling in HIV-1}, issn = {1438-0064}, doi = {10.4310/CMS.2014.v12.n5.a4}, url = {http://nbn-resolving.de/urn:nbn:de:0297-zib-41955}, year = {2013}, abstract = {Markov Decision Processes (MDP) or Partially Observable MDPs (POMDP) are used for modelling situations in which the evolution of a process is partly random and partly controllable. These MDP theories allow for computing the optimal control policy for processes that can continuously or frequently be observed, even if only partially. However, they cannot be applied if state observation is very costly and therefore rare (in time). We present a novel MDP theory for rare, costly observations and derive the corresponding Bellman equation. In the new theory, state information can be derived for a particular cost after certain, rather long time intervals. The resulting information costs enter into the total cost and thus into the optimization criterion. This approach applies to many real world problems, particularly in the medical context, where the medical condition is examined rather rarely because examination costs are high. At the same time, the approach allows for efficient numerical realization. We demonstrate the usefulness of the novel theory by determining, from the national economic perspective, optimal therapeutic policies for the treatment of the human immunodefficiency virus (HIV) in resource-rich and resource-poor settings. Based on the developed theory and models, we discover that available drugs may not be utilized efficiently in resource-poor settings due to exorbitant diagnostic costs.}, language = {en} } @misc{WangHartmannSchuette2013, author = {Wang, Han and Hartmann, Carsten and Sch{\"u}tte, Christof}, title = {Linear response theory and optimal control for a molecular system under nonequilibrium conditions}, url = {http://nbn-resolving.de/urn:nbn:de:0297-zib-18944}, year = {2013}, abstract = {In this paper, we propose a straightforward generalization of linear response theory to systems in nonequilibrium that are subject to nonequilibrium driving. We briefly revisit the standard linear response result for equilibrium systems, where we consider Langevin dynamics as a special case, and then give an alternative derivation using a change-of-measure argument that does not rely on any stationarity or reversibility assumption. This procedure moreover easily enables us to calculate the second order correction to the linear response formula (which may or may not be useful in practice). Furthermore, we outline how the novel nonequilibirum linear response formula can be used to compute optimal controls of molecular systems for cases in which one wants to steer the system to maximize a certain target expectation value. We illustrate our approach with simple numerical examples.}, language = {en} } @misc{ZhangWangHartmannetal.2014, author = {Zhang, Wei and Wang, Han and Hartmann, Carsten and Weber, Marcus and Sch{\"u}tte, Christof}, title = {Applications of the cross-entropy method to importance sampling and optimal control of diffusions}, issn = {1438-0064}, doi = {10.1137/14096493X}, url = {http://nbn-resolving.de/urn:nbn:de:0297-zib-49720}, year = {2014}, abstract = {We study the cross-entropy method for diffusions. One of the results is a versatile cross-entropy algorithm that can be used to design efficient importance sampling strategies for rare events or to solve optimal control problems. The approach is based on the minimization of a suitable cross-entropy functional, with a parametric family of exponentially tilted probability distributions. We illustrate the new algorithm with several numerical examples and discuss algorithmic issues and possible extensions of the method.}, language = {en} } @misc{HartmannBanischSarichetal.2013, author = {Hartmann, Carsten and Banisch, Ralf and Sarich, Marco and Badowski, Thomas and Sch{\"u}tte, Christof}, title = {Characterization of Rare Events in Molecular Dynamics}, issn = {1438-0064}, url = {http://nbn-resolving.de/urn:nbn:de:0297-zib-42410}, year = {2013}, abstract = {A good deal of molecular dynamics simulations aims at predicting and quantifying rare events, such as the folding of a protein or a phase transition. Simulating rare events is often prohibitive, especially if the equations of motion are high-dimensional, as is the case in molecular dynamics. Various algorithms have been proposed for efficiently computing mean first passage times, transition rates or reaction pathways. This article surveys and discusses recent developments in the field of rare event simulation and outlines a new approach that combines ideas from optimal control and statistical mechanics. The optimal control approach described in detail resembles the use of Jarzynski's equality for free energy calculations, but with an optimized protocol that speeds up the sampling, while (theoretically) giving variance-free estimators of the rare events statistics. We illustrate the new approach with two numerical examples and discuss its relation to existing methods.}, language = {en} } @misc{SarichBanischHartmannetal.2013, author = {Sarich, Marco and Banisch, Ralf and Hartmann, Carsten and Sch{\"u}tte, Christof}, title = {Markov State Models for Rare Events in Molecular Dynamics}, issn = {1438-0064}, url = {http://nbn-resolving.de/urn:nbn:de:0297-zib-42420}, year = {2013}, abstract = {Rare but important transition events between long lived states are a key feature of many molecular systems. In many cases the computation of rare event statistics by direct molecular dynamics (MD) simulations is infeasible even on the most powerful computers because of the immensely long simulation timescales needed. Recently a technique for spatial discretization of the molecular state space designed to help overcome such problems, so-called Markov State Models (MSMs), has attracted a lot of attention. We review the theoretical background and algorithmic realization of MSMs and illustrate their use by some numerical examples. Furthermore we introduce a novel approach to using MSMs for the efficient solution of optimal control problems that appear in applications where one desires to optimize molecular properties by means of external controls.}, language = {en} }