@inproceedings{RitzRatkePhanetal.2021, author = {Ritz, Fabian and Ratke, Daniel and Phan, Thomy and Belzner, Lenz and Linnhoff-Popien, Claudia}, title = {A sustainable ecosystem through emergent cooperation in multi-agent reinforcement learning}, volume = {2021}, pages = {isal_a_00399}, booktitle = {Proceedings of the Artificial Life Conference 2021}, publisher = {MIT Press}, address = {Cambridge}, doi = {https://doi.org/10.1162/isal_a_00399}, year = {2021}, language = {en} } @inproceedings{SchmidBelznerLinnhoffPopien2021, author = {Schmid, Kyrill and Belzner, Lenz and Linnhoff-Popien, Claudia}, title = {Learning to penalize other learning agents}, volume = {2021}, pages = {isal_a_00369}, booktitle = {Proceedings of the Artificial Life Conference 2021}, publisher = {MIT Press}, address = {Cambridge}, doi = {https://doi.org/10.1162/isal_a_00369}, year = {2021}, language = {en} } @article{GaborSedlmeierPhanetal.2020, author = {Gabor, Thomas and Sedlmeier, Andreas and Phan, Thomy and Ritz, Fabian and Kiermeier, Marie and Belzner, Lenz and Kempter, Bernhard and Klein, Cornel and Sauer, Horst and Schmid, Reiner and Wieghardt, Jan and Zeller, Marc and Linnhoff-Popien, Claudia}, title = {The scenario coevolution paradigm}, volume = {22}, journal = {International Journal on Software Tools for Technology Transfer}, subtitle = {adaptive quality assurance for adaptive systems}, number = {4}, publisher = {Springer}, address = {Berlin}, issn = {1433-2787}, doi = {https://doi.org/10.1007/s10009-020-00560-5}, pages = {457 -- 476}, year = {2020}, abstract = {Systems are becoming increasingly more adaptive, using techniques like machine learning to enhance their behavior on their own rather than only through human developers programming them. We analyze the impact the advent of these new techniques has on the discipline of rigorous software engineering, especially on the issue of quality assurance. To this end, we provide a general description of the processes related to machine learning and embed them into a formal framework for the analysis of adaptivity, recognizing that to test an adaptive system a new approach to adaptive testing is necessary. We introduce scenario coevolution as a design pattern describing how system and test can work as antagonists in the process of software evolution. While the general pattern applies to large-scale processes (including human developers further augmenting the system), we show all techniques on a smaller-scale example of an agent navigating a simple smart factory. We point out new aspects in software engineering for adaptive systems that may be tackled naturally using scenario coevolution. This work is a substantially extended take on Gabor et al. (International symposium on leveraging applications of formal methods, Springer, pp 137-154, 2018).}, language = {en} } @article{PhanSommerRitzetal.2024, author = {Phan, Thomy and Sommer, Felix and Ritz, Fabian and Altmann, Philipp and N{\"u}ßlein, Jonas and K{\"o}lle, Michael and Belzner, Lenz and Linnhoff-Popien, Claudia}, title = {Emergent cooperation from mutual acknowledgment exchange in multi-agent reinforcement learning}, volume = {38}, pages = {34}, journal = {Autonomous Agents and Multi-Agent Systems}, number = {2}, publisher = {Springer}, address = {Dordrecht}, issn = {1573-7454}, doi = {https://doi.org/10.1007/s10458-024-09666-5}, year = {2024}, abstract = {Peer incentivization (PI) is a recent approach where all agents learn to reward or penalize each other in a distributed fashion, which often leads to emergent cooperation. Current PI mechanisms implicitly assume a flawless communication channel in order to exchange rewards. These rewards are directly incorporated into the learning process without any chance to respond with feedback. Furthermore, most PI approaches rely on global information, which limits scalability and applicability to real-world scenarios where only local information is accessible. In this paper, we propose Mutual Acknowledgment Token Exchange (MATE), a PI approach defined by a two-phase communication protocol to exchange acknowledgment tokens as incentives to shape individual rewards mutually. All agents condition their token transmissions on the locally estimated quality of their own situations based on environmental rewards and received tokens. MATE is completely decentralized and only requires local communication and information. We evaluate MATE in three social dilemma domains. Our results show that MATE is able to achieve and maintain significantly higher levels of cooperation than previous PI approaches. In addition, we evaluate the robustness of MATE in more realistic scenarios, where agents can deviate from the protocol and communication failures can occur. We also evaluate the sensitivity of MATE w.r.t. the choice of token values.}, language = {en} } @unpublished{PhanSommerRitzetal.2022, author = {Phan, Thomy and Sommer, Felix and Ritz, Fabian and Altmann, Philipp and N{\"u}ßlein, Jonas and K{\"o}lle, Michael and Belzner, Lenz and Linnhoff-Popien, Claudia}, title = {Emergent Cooperation from Mutual Acknowledgment Exchange in Multi-Agent Reinforcement Learning}, titleParent = {Research Square}, publisher = {Research Square}, address = {Durham}, doi = {https://doi.org/10.21203/rs.3.rs-2315844/v1}, year = {2022}, abstract = {Peer incentivization (PI) is a recent approach, where all agents learn to reward or to penalize each other in a distributed fashion which often leads to emergent cooperation. Current PI mechanisms implicitly assume a flawless communication channel in order to exchange rewards. These rewards are directly integrated into the learning process without any chance to respond with feedback. Furthermore, most PI approaches rely on global information which limits scalability and applicability to real-world scenarios, where only local information is accessible. In this paper, we propose Mutual Acknowledgment Token Exchange (MATE), a PI approach defined by a two-phase communication protocol to mutually exchange acknowledgment tokens to shape individual rewards. Each agent evaluates the monotonic improvement of its individual situation in order to accept or reject acknowledgment requests from other agents. MATE is completely decentralized and only requires local communication and information. We evaluate MATE in three social dilemma domains. Our results show that MATE is able to achieve and maintain significantly higher levels of cooperation than previous PI approaches. In addition, we evaluate the robustness of MATE in more realistic scenarios, where agents can defect from the protocol and where communication failures can occur. We also evaluate the sensitivity of MATE w.r.t. the choice of token values.}, language = {en} } @inproceedings{SchmidBelznerPhanetal.2020, author = {Schmid, Kyrill and Belzner, Lenz and Phan, Thomy and Gabor, Thomas and Linnhoff-Popien, Claudia}, title = {Multi-agent reinforcement learning for bargaining under risk and asymmetric information}, booktitle = {Proceedings of the 12th International Conference on Agents and Artificial Intelligence - Volume 1: ICAART 2020}, editor = {Rocha, Ana Paula and Steels, Luc and Herik, Jaap van den}, publisher = {SciTePress}, address = {Set{\´u}bal}, isbn = {978-989-758-395-7}, issn = {2184-433X}, doi = {https://doi.org/10.5220/0008913901440151}, pages = {144 -- 151}, year = {2020}, abstract = {In cooperative game theory bargaining games refer to situations where players can agree to any one of a variety of outcomes but there is a conflict on which specific outcome to choose. However, the players cannot impose a specific outcome on others and if no agreement is reached all players receive a predetermined status quo outcome. Bargaining games have been studied from a variety of fields, including game theory, economics, psychology and simulation based methods like genetic algorithms. In this work we extend the analysis by means of deep multi-agent reinforcement learning (MARL). To study the dynamics of bargaining with reinforcement learning we propose two different bargaining environments which display the following situations: in the first domain two agents have to agree on the division of an asset, e.g., the division of a fixed amount of money between each other. The second domain models a seller-buyer scenario in which agents must agree on a price for a product. We empirica lly demonstrate that the bargaining result under MARL is influenced by agents' risk-aversion as well as information asymmetry between agents.}, language = {en} }