@article{BiemannGunkelSchelleretal., author = {Biemann, Marco and Gunkel, Philipp Andreas and Scheller, Fabian and Huang, Lizhen and Liu, Xiufeng}, title = {Data Center HVAC Control Harnessing Flexibility Potential via Real-Time Pricing Cost Optimization Using Reinforcement Learning}, series = {IEEE Internet of Things Journal}, volume = {10}, journal = {IEEE Internet of Things Journal}, number = {15}, doi = {10.1109/jiot.2023.3263261}, pages = {13876 -- 13894}, abstract = {With increasing electricity prices, cost savings through load shifting are becoming increasingly important for energy end users. While dynamic pricing encourages customers to shift demand to low price periods, the nonstationary and highly volatile nature of electricity prices poses a significant challenge to energy management systems. In this article, we investigate the flexibility potential of data centers by optimizing heating, ventilation, and air conditioning systems with a general model-free reinforcement learning (RL) approach. Since the soft actor-critic algorithm with feedforward networks did not work satisfactorily in this scenario, we propose instead a parameterization with a recurrent neural network architecture to successfully handle spot-market price data. The past is encoded into a hidden state, which provides a way to learn the temporal dependencies in the observations and highly volatile rewards. The proposed method is then evaluated in experiments on a simulated data center. Considering real temperature and price signals over multiple years, the results show a cost reduction compared to a proportional, integral and derivative controller while maintaining the temperature of the data center within the desired operating ranges. In this context, this work demonstrates an innovative and applicable RL approach that incorporates complex economic objectives into agent decision-making. The proposed control method can be integrated into various Internet of Things-based smart building solutions for energy management.}, language = {en} } @article{BiemannSchellerLiuetal., author = {Biemann, Marco and Scheller, Fabian and Liu, Xiufeng and Huang, Lizhen}, title = {Experimental evaluation of model-free reinforcement learning algorithms for continuous HVAC control}, series = {Applied Energy}, volume = {298}, journal = {Applied Energy}, issn = {03062619}, doi = {10.1016/j.apenergy.2021.117164}, pages = {117164 -- 117164}, abstract = {Controlling heating, ventilation and air-conditioning (HVAC) systems is crucial to improving demand-side energy efficiency. At the same time, the thermodynamics of buildings and uncertainties regarding human activities make effective management challenging. While the concept of model-free reinforcement learning demonstrates various advantages over existing strategies, the literature relies heavily on value-based methods that can hardly handle complex HVAC systems. This paper conducts experiments to evaluate four actor-critic algorithms in a simulated data centre. The performance evaluation is based on their ability to maintain thermal stability while increasing energy efficiency and on their adaptability to weather dynamics. Because of the enormous significance of practical use, special attention is paid to data efficiency. Compared to the model-based controller implemented into EnergyPlus, all applied algorithms can reduce energy consumption by at least 10\% by simultaneously keeping the hourly average temperature in the desired range. Robustness tests in terms of different reward functions and weather conditions verify these results. With increasing training, we also see a smaller trade-off between thermal stability and energy reduction. Thus, the Soft Actor Critic algorithm achieves a stable performance with ten times less data than on-policy methods. In this regard, we recommend using this algorithm in future experiments, due to both its interesting theoretical properties and its practical results.}, language = {en} }