<?xml version="1.0" encoding="utf-8"?>
<export-example>
  <doc>
    <id>1297</id>
    <completedYear/>
    <publishedYear>2021</publishedYear>
    <thesisYearAccepted/>
    <language>eng</language>
    <pageFirst/>
    <pageLast/>
    <pageNumber>10</pageNumber>
    <edition/>
    <issue/>
    <volume>2021</volume>
    <articleNumber>isal_a_00399</articleNumber>
    <type>conferenceobject</type>
    <publisherName>MIT Press</publisherName>
    <publisherPlace>Cambridge</publisherPlace>
    <creatingCorporation/>
    <contributingCorporation/>
    <belongsToBibliography>1</belongsToBibliography>
    <completedDate>2022-02-17</completedDate>
    <publishedDate>--</publishedDate>
    <thesisDateAccepted>--</thesisDateAccepted>
    <title language="eng">A sustainable ecosystem through emergent cooperation in multi-agent reinforcement learning</title>
    <parentTitle language="eng">Proceedings of the Artificial Life Conference 2021</parentTitle>
    <identifier type="urn">urn:nbn:de:bvb:573-12975</identifier>
    <enrichment key="THI_review">peer-review</enrichment>
    <enrichment key="THI_openaccess">ja</enrichment>
    <enrichment key="THI_conferenceName">ALIFE 2021: The 2021 Conference on Artificial Life, online, 19.-23.07.2021</enrichment>
    <enrichment key="opus.source">publish</enrichment>
    <enrichment key="THI_relatedIdentifier">https://doi.org/10.1162/isal_a_00399</enrichment>
    <licence>Creative Commons BY 4.0</licence>
    <author>
      <first_name>Fabian</first_name>
      <last_name>Ritz</last_name>
    </author>
    <author>
      <first_name>Daniel</first_name>
      <last_name>Ratke</last_name>
    </author>
    <author>
      <first_name>Thomy</first_name>
      <last_name>Phan</last_name>
    </author>
    <author>
      <first_name>Lenz</first_name>
      <last_name>Belzner</last_name>
    </author>
    <author>
      <first_name>Claudia</first_name>
      <last_name>Linnhoff-Popien</last_name>
    </author>
    <collection role="open_access" number="">open_access</collection>
    <collection role="institutes" number="19311">Fakultät Elektro- und Informationstechnik</collection>
    <collection role="institutes" number="19379">AImotion Bavaria</collection>
    <collection role="persons" number="43324">Belzner, Lenz</collection>
    <thesisPublisher>Technische Hochschule Ingolstadt</thesisPublisher>
    <file>https://opus4.kobv.de/opus4-haw/files/1297/isal_a_00399.pdf</file>
  </doc>
  <doc>
    <id>1298</id>
    <completedYear/>
    <publishedYear>2021</publishedYear>
    <thesisYearAccepted/>
    <language>eng</language>
    <pageFirst/>
    <pageLast/>
    <pageNumber>8</pageNumber>
    <edition/>
    <issue/>
    <volume/>
    <articleNumber/>
    <type>conferenceobject</type>
    <publisherName>IEEE</publisherName>
    <publisherPlace>Piscataway</publisherPlace>
    <creatingCorporation/>
    <contributingCorporation/>
    <belongsToBibliography>0</belongsToBibliography>
    <completedDate>2022-02-17</completedDate>
    <publishedDate>--</publishedDate>
    <thesisDateAccepted>--</thesisDateAccepted>
    <title language="eng">Distributed emergent agreements with deep reinforcement learning</title>
    <parentTitle language="eng">2021 International Joint Conference on Neural Networks&#13;
(IJCNN) Proceedings</parentTitle>
    <identifier type="isbn">978-1-6654-3900-8</identifier>
    <identifier type="issn">2161-4407</identifier>
    <enrichment key="THI_review">peer-review</enrichment>
    <enrichment key="THI_openaccess">nein</enrichment>
    <enrichment key="THI_conferenceName">International Joint Conference on Neural Networks (IJCNN), online, 18.-22.07.2021</enrichment>
    <enrichment key="opus.source">publish</enrichment>
    <enrichment key="THI_relatedIdentifier">https://doi.org/10.1109/IJCNN52387.2021.9533333</enrichment>
    <author>
      <first_name>Kyrill</first_name>
      <last_name>Schmid</last_name>
    </author>
    <author>
      <first_name>Robert</first_name>
      <last_name>Müller</last_name>
    </author>
    <author>
      <first_name>Lenz</first_name>
      <last_name>Belzner</last_name>
    </author>
    <author>
      <first_name>Johannes</first_name>
      <last_name>Tochtermann</last_name>
    </author>
    <author>
      <first_name>Claudia</first_name>
      <last_name>Linnhoff-Popien</last_name>
    </author>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>Waste materials</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>Neural networks</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>Buildings</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>Reinforcement learning</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>Production facilities</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>Robustness</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>Autonomous agents</value>
    </subject>
    <collection role="persons" number="43324">Belzner, Lenz</collection>
  </doc>
  <doc>
    <id>1295</id>
    <completedYear/>
    <publishedYear>2021</publishedYear>
    <thesisYearAccepted/>
    <language>eng</language>
    <pageFirst/>
    <pageLast/>
    <pageNumber>15</pageNumber>
    <edition/>
    <issue/>
    <volume/>
    <articleNumber/>
    <type>conferenceobject</type>
    <publisherName>Neural Information Processing Systems Foundation, Inc. (NIPS)</publisherName>
    <publisherPlace/>
    <creatingCorporation/>
    <contributingCorporation/>
    <belongsToBibliography>1</belongsToBibliography>
    <completedDate>2022-02-17</completedDate>
    <publishedDate>--</publishedDate>
    <thesisDateAccepted>--</thesisDateAccepted>
    <title language="eng">VAST: Value Function Factorization with Variable Agent Sub-Teams</title>
    <parentTitle language="eng">Advances in Neural Information Processing Systems 34 (NeurIPS 2021)</parentTitle>
    <identifier type="url">https://proceedings.neurips.cc/paper/2021/hash/c97e7a5153badb6576d8939469f58336-Abstract.html</identifier>
    <enrichment key="THI_review">peer-review</enrichment>
    <enrichment key="THI_openaccess">nein</enrichment>
    <enrichment key="opus.source">publish</enrichment>
    <enrichment key="THI_conferenceName">35th Conference on Neural Information Processing Systems (NeurIPS 2021), virtuell, 06.-14. Dezember 2021</enrichment>
    <enrichment key="THI_DownloadUrl">https://proceedings.neurips.cc/paper/2021/file/c97e7a5153badb6576d8939469f58336-Supplemental.pdf</enrichment>
    <author>
      <first_name>Thomy</first_name>
      <last_name>Phan</last_name>
    </author>
    <author>
      <first_name>Fabian</first_name>
      <last_name>Ritz</last_name>
    </author>
    <author>
      <first_name>Lenz</first_name>
      <last_name>Belzner</last_name>
    </author>
    <author>
      <first_name>Philipp</first_name>
      <last_name>Altmann</last_name>
    </author>
    <author>
      <first_name>Thomas</first_name>
      <last_name>Gabor</last_name>
    </author>
    <author>
      <first_name>Claudia</first_name>
      <last_name>Linnhoff-Popien</last_name>
    </author>
    <subject>
      <language>deu</language>
      <type>uncontrolled</type>
      <value>Multi-Agent Learning</value>
    </subject>
    <subject>
      <language>deu</language>
      <type>uncontrolled</type>
      <value>Reinforcement Learning</value>
    </subject>
    <subject>
      <language>deu</language>
      <type>uncontrolled</type>
      <value>Value Function Factorization</value>
    </subject>
    <collection role="institutes" number="19311">Fakultät Elektro- und Informationstechnik</collection>
    <collection role="institutes" number="19379">AImotion Bavaria</collection>
    <collection role="persons" number="43324">Belzner, Lenz</collection>
  </doc>
  <doc>
    <id>1296</id>
    <completedYear/>
    <publishedYear>2021</publishedYear>
    <thesisYearAccepted/>
    <language>eng</language>
    <pageFirst/>
    <pageLast/>
    <pageNumber>9</pageNumber>
    <edition/>
    <issue/>
    <volume>2021</volume>
    <articleNumber>isal_a_00369</articleNumber>
    <type>conferenceobject</type>
    <publisherName>MIT Press</publisherName>
    <publisherPlace>Cambridge</publisherPlace>
    <creatingCorporation/>
    <contributingCorporation/>
    <belongsToBibliography>0</belongsToBibliography>
    <completedDate>2022-02-17</completedDate>
    <publishedDate>--</publishedDate>
    <thesisDateAccepted>--</thesisDateAccepted>
    <title language="eng">Learning to penalize other learning agents</title>
    <parentTitle language="eng">Proceedings of the Artificial Life Conference 2021</parentTitle>
    <identifier type="urn">urn:nbn:de:bvb:573-12966</identifier>
    <enrichment key="THI_review">peer-review</enrichment>
    <enrichment key="THI_openaccess">ja</enrichment>
    <enrichment key="THI_conferenceName">ALIFE 2021: The 2021 Conference on Artificial Life, online, 19.-23.07.2021</enrichment>
    <enrichment key="opus.source">publish</enrichment>
    <enrichment key="THI_relatedIdentifier">https://doi.org/10.1162/isal_a_00369</enrichment>
    <licence>Creative Commons BY 4.0</licence>
    <author>
      <first_name>Kyrill</first_name>
      <last_name>Schmid</last_name>
    </author>
    <author>
      <first_name>Lenz</first_name>
      <last_name>Belzner</last_name>
    </author>
    <author>
      <first_name>Claudia</first_name>
      <last_name>Linnhoff-Popien</last_name>
    </author>
    <collection role="open_access" number="">open_access</collection>
    <collection role="persons" number="43324">Belzner, Lenz</collection>
    <thesisPublisher>Technische Hochschule Ingolstadt</thesisPublisher>
    <file>https://opus4.kobv.de/opus4-haw/files/1296/isal_a_00369.pdf</file>
  </doc>
  <doc>
    <id>2578</id>
    <completedYear/>
    <publishedYear>2022</publishedYear>
    <thesisYearAccepted/>
    <language>eng</language>
    <pageFirst>1047</pageFirst>
    <pageLast>1055</pageLast>
    <pageNumber/>
    <edition/>
    <issue/>
    <volume/>
    <articleNumber/>
    <type>conferenceobject</type>
    <publisherName>International Foundation for Autonomous Agents and Multiagent Systems</publisherName>
    <publisherPlace>Richland</publisherPlace>
    <creatingCorporation/>
    <contributingCorporation/>
    <belongsToBibliography>1</belongsToBibliography>
    <completedDate>2022-07-28</completedDate>
    <publishedDate>--</publishedDate>
    <thesisDateAccepted>--</thesisDateAccepted>
    <title language="eng">Emergent Cooperation from Mutual Acknowledgment Exchange</title>
    <parentTitle language="eng">AAMAS '22: Proceedings of the 21st International Conference on Autonomous Agents and Multiagent Systems</parentTitle>
    <identifier type="isbn">978-1-4503-9213-6</identifier>
    <enrichment key="THI_review">peer-review</enrichment>
    <enrichment key="THI_openaccess">nein</enrichment>
    <enrichment key="THI_conferenceName">AAMAS ' 22: International Conference on Autonomous Agents and Multi-Agent Systems, online, 09.-13.05.2022</enrichment>
    <enrichment key="opus.source">publish</enrichment>
    <enrichment key="THI_relatedIdentifier">https://dl.acm.org/doi/10.5555/3535850.3535967</enrichment>
    <author>
      <first_name>Thomy</first_name>
      <last_name>Phan</last_name>
    </author>
    <author>
      <first_name>Felix</first_name>
      <last_name>Sommer</last_name>
    </author>
    <author>
      <first_name>Philipp</first_name>
      <last_name>Altmann</last_name>
    </author>
    <author>
      <first_name>Fabian</first_name>
      <last_name>Ritz</last_name>
    </author>
    <author>
      <first_name>Lenz</first_name>
      <last_name>Belzner</last_name>
    </author>
    <author>
      <first_name>Claudia</first_name>
      <last_name>Linnhoff-Popien</last_name>
    </author>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>multi-agent learning</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>reinforcement learning</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>mutual acknowledgments</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>peer incentivization</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>emergent cooperation</value>
    </subject>
    <collection role="institutes" number="19311">Fakultät Elektro- und Informationstechnik</collection>
    <collection role="institutes" number="19379">AImotion Bavaria</collection>
    <collection role="persons" number="43324">Belzner, Lenz</collection>
  </doc>
  <doc>
    <id>1398</id>
    <completedYear/>
    <publishedYear>2019</publishedYear>
    <thesisYearAccepted/>
    <language>eng</language>
    <pageFirst>7941</pageFirst>
    <pageLast>7948</pageLast>
    <pageNumber/>
    <edition/>
    <issue>1</issue>
    <volume>33</volume>
    <articleNumber/>
    <type>conferenceobject</type>
    <publisherName>AAAI Press</publisherName>
    <publisherPlace>Palo Alto</publisherPlace>
    <creatingCorporation/>
    <contributingCorporation/>
    <belongsToBibliography>0</belongsToBibliography>
    <completedDate>2022-02-22</completedDate>
    <publishedDate>--</publishedDate>
    <thesisDateAccepted>--</thesisDateAccepted>
    <title language="eng">Memory Bounded Open-Loop Planning in Large POMDPs Using Thompson Sampling</title>
    <parentTitle language="eng">AAAI-19 / IAAI-19 / EAAI-20 Proceedings</parentTitle>
    <identifier type="issn">2374-3468</identifier>
    <enrichment key="THI_review">peer-review</enrichment>
    <enrichment key="THI_openaccess">nein</enrichment>
    <enrichment key="opus.source">publish</enrichment>
    <enrichment key="THI_relatedIdentifier">https://doi.org/10.1609/aaai.v33i01.33017941</enrichment>
    <enrichment key="THI_conferenceName">The Thirty-Third AAAI Conference on Artificial Intelligence (AAAI-19), Honolulu (USA), 27.01. - 01.02.2019</enrichment>
    <author>
      <first_name>Thomy</first_name>
      <last_name>Phan</last_name>
    </author>
    <author>
      <first_name>Lenz</first_name>
      <last_name>Belzner</last_name>
    </author>
    <author>
      <first_name>Marie</first_name>
      <last_name>Kiermeier</last_name>
    </author>
    <author>
      <first_name>Markus</first_name>
      <last_name>Friedrich</last_name>
    </author>
    <author>
      <first_name>Kyrill</first_name>
      <last_name>Schmid</last_name>
    </author>
    <author>
      <first_name>Claudia</first_name>
      <last_name>Linnhoff-Popien</last_name>
    </author>
    <collection role="persons" number="43324">Belzner, Lenz</collection>
  </doc>
  <doc>
    <id>1399</id>
    <completedYear/>
    <publishedYear>2019</publishedYear>
    <thesisYearAccepted/>
    <language>eng</language>
    <pageFirst>424</pageFirst>
    <pageLast>431</pageLast>
    <pageNumber/>
    <edition/>
    <issue/>
    <volume/>
    <articleNumber>isal_a_00197</articleNumber>
    <type>conferenceobject</type>
    <publisherName>MIT Press</publisherName>
    <publisherPlace>Cambridge</publisherPlace>
    <creatingCorporation/>
    <contributingCorporation/>
    <belongsToBibliography>0</belongsToBibliography>
    <completedDate>2022-02-22</completedDate>
    <publishedDate>--</publishedDate>
    <thesisDateAccepted>--</thesisDateAccepted>
    <title language="eng">Self-Replication in Neural Networks</title>
    <parentTitle language="eng">Artificial Life Conference Proceedings</parentTitle>
    <enrichment key="THI_review">peer-review</enrichment>
    <enrichment key="THI_openaccess">nein</enrichment>
    <enrichment key="opus.source">publish</enrichment>
    <enrichment key="THI_conferenceName">2019 Conference on Artificial Life (ALIFE 2019), Newcastle upon Tyne (UK), 29.07. - 02.08.2019</enrichment>
    <enrichment key="THI_relatedIdentifier">https://doi.org/10.1162/isal_a_00197</enrichment>
    <author>
      <first_name>Thomas</first_name>
      <last_name>Gabor</last_name>
    </author>
    <author>
      <first_name>Steffen</first_name>
      <last_name>Illium</last_name>
    </author>
    <author>
      <first_name>Andy</first_name>
      <last_name>Mattausch</last_name>
    </author>
    <author>
      <first_name>Lenz</first_name>
      <last_name>Belzner</last_name>
    </author>
    <author>
      <first_name>Claudia</first_name>
      <last_name>Linnhoff-Popien</last_name>
    </author>
    <collection role="persons" number="43324">Belzner, Lenz</collection>
  </doc>
  <doc>
    <id>1400</id>
    <completedYear/>
    <publishedYear>2019</publishedYear>
    <thesisYearAccepted/>
    <language>eng</language>
    <pageFirst>598</pageFirst>
    <pageLast>605</pageLast>
    <pageNumber/>
    <edition/>
    <issue/>
    <volume/>
    <articleNumber>isal_a_00226</articleNumber>
    <type>conferenceobject</type>
    <publisherName>MIT Press</publisherName>
    <publisherPlace>Cambridge</publisherPlace>
    <creatingCorporation/>
    <contributingCorporation/>
    <belongsToBibliography>0</belongsToBibliography>
    <completedDate>2022-02-22</completedDate>
    <publishedDate>--</publishedDate>
    <thesisDateAccepted>--</thesisDateAccepted>
    <title language="eng">Emergent Escape-based Flocking behavior using Multi-Agent Reinforcement Learning</title>
    <parentTitle language="eng">Artificial Life Conference Proceedings</parentTitle>
    <enrichment key="THI_review">peer-review</enrichment>
    <enrichment key="THI_openaccess">nein</enrichment>
    <enrichment key="opus.source">publish</enrichment>
    <enrichment key="THI_relatedIdentifier">https://doi.org/10.1162/isal_a_00226</enrichment>
    <enrichment key="THI_conferenceName">ALIFE 2019: The 2019 Conference on Artificial Life, Newcastle upon Tyne (UK), 29.07. - 02.08.2019</enrichment>
    <author>
      <first_name>Carsten</first_name>
      <last_name>Hahn</last_name>
    </author>
    <author>
      <first_name>Thomy</first_name>
      <last_name>Phan</last_name>
    </author>
    <author>
      <first_name>Thomas</first_name>
      <last_name>Gabor</last_name>
    </author>
    <author>
      <first_name>Lenz</first_name>
      <last_name>Belzner</last_name>
    </author>
    <author>
      <first_name>Claudia</first_name>
      <last_name>Linnhoff-Popien</last_name>
    </author>
    <collection role="persons" number="43324">Belzner, Lenz</collection>
  </doc>
  <doc>
    <id>1401</id>
    <completedYear/>
    <publishedYear>2019</publishedYear>
    <thesisYearAccepted/>
    <language>eng</language>
    <pageFirst>2162</pageFirst>
    <pageLast>2164</pageLast>
    <pageNumber/>
    <edition/>
    <issue/>
    <volume/>
    <articleNumber/>
    <type>conferenceobject</type>
    <publisherName>International Foundation for Autonomous Agents and Multiagent Systems</publisherName>
    <publisherPlace>Richland</publisherPlace>
    <creatingCorporation/>
    <contributingCorporation/>
    <belongsToBibliography>0</belongsToBibliography>
    <completedDate>2022-02-22</completedDate>
    <publishedDate>--</publishedDate>
    <thesisDateAccepted>--</thesisDateAccepted>
    <title language="eng">Distributed Policy Iteration for Scalable Approximation of Cooperative Multi-Agent Policies</title>
    <parentTitle language="eng">AAMAS '19: Proceedings of the 18th International Conference on Autonomous Agents and MultiAgent Systems</parentTitle>
    <identifier type="isbn">978-1-4503-6309-9</identifier>
    <identifier type="url">https://www.ifaamas.org/Proceedings/aamas2019/forms/contents.htm</identifier>
    <enrichment key="THI_review">peer-review</enrichment>
    <enrichment key="THI_openaccess">nein</enrichment>
    <enrichment key="opus.source">publish</enrichment>
    <enrichment key="THI_conferenceName">AAMAS '19: International Conference on Autonomous Agents and Multiagent Systems, Montreal (Canada), 13.-17.05.2019</enrichment>
    <author>
      <first_name>Thomy</first_name>
      <last_name>Phan</last_name>
    </author>
    <author>
      <first_name>Kyrill</first_name>
      <last_name>Schmid</last_name>
    </author>
    <author>
      <first_name>Lenz</first_name>
      <last_name>Belzner</last_name>
    </author>
    <author>
      <first_name>Thomas</first_name>
      <last_name>Gabor</last_name>
    </author>
    <author>
      <first_name>Sebastian</first_name>
      <last_name>Feld</last_name>
    </author>
    <author>
      <first_name>Claudia</first_name>
      <last_name>Linnhoff-Popien</last_name>
    </author>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>multi-agent planning</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>multi-agent learning</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>policy iteration</value>
    </subject>
    <collection role="persons" number="43324">Belzner, Lenz</collection>
  </doc>
  <doc>
    <id>1391</id>
    <completedYear/>
    <publishedYear>2021</publishedYear>
    <thesisYearAccepted/>
    <language>eng</language>
    <pageFirst>11308</pageFirst>
    <pageLast>11316</pageLast>
    <pageNumber/>
    <edition/>
    <issue>13</issue>
    <volume>35</volume>
    <articleNumber/>
    <type>conferenceobject</type>
    <publisherName>AAAI Press</publisherName>
    <publisherPlace>Palo Alto (CA)</publisherPlace>
    <creatingCorporation/>
    <contributingCorporation/>
    <belongsToBibliography>0</belongsToBibliography>
    <completedDate>2022-02-22</completedDate>
    <publishedDate>--</publishedDate>
    <thesisDateAccepted>--</thesisDateAccepted>
    <title language="eng">Resilient multi-agent reinforcement learning with adversarial value decomposition</title>
    <parentTitle language="eng">AAAI-21 / IAAI-21 / EAAI-21 Proceedings</parentTitle>
    <identifier type="issn">2374-3468</identifier>
    <identifier type="isbn">978-1-57735-866-4</identifier>
    <enrichment key="THI_review">peer-review</enrichment>
    <enrichment key="THI_openaccess">nein</enrichment>
    <enrichment key="opus.source">publish</enrichment>
    <enrichment key="THI_relatedIdentifier">https://ojs.aaai.org/index.php/AAAI/article/view/17348</enrichment>
    <enrichment key="THI_conferenceName">The Thirty-Fifth AAAI Conference on Artificial Intelligence (AAAI-21), onlne, 02.-09.02.2021</enrichment>
    <author>
      <first_name>Thomy</first_name>
      <last_name>Phan</last_name>
    </author>
    <author>
      <first_name>Lenz</first_name>
      <last_name>Belzner</last_name>
    </author>
    <author>
      <first_name>Thomas</first_name>
      <last_name>Gabor</last_name>
    </author>
    <author>
      <first_name>Andreas</first_name>
      <last_name>Sedlmeier</last_name>
    </author>
    <author>
      <first_name>Fabian</first_name>
      <last_name>Ritz</last_name>
    </author>
    <author>
      <first_name>Claudia</first_name>
      <last_name>Linnhoff-Popien</last_name>
    </author>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>Multiagent Learning</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>Adversarial Learning &amp; Robustness</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>Adversarial Agents</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>Reinforcement Learning</value>
    </subject>
    <collection role="persons" number="43324">Belzner, Lenz</collection>
  </doc>
  <doc>
    <id>1392</id>
    <completedYear/>
    <publishedYear>2020</publishedYear>
    <thesisYearAccepted/>
    <language>eng</language>
    <pageFirst>457</pageFirst>
    <pageLast>476</pageLast>
    <pageNumber/>
    <edition/>
    <issue>4</issue>
    <volume>22</volume>
    <articleNumber/>
    <type>article</type>
    <publisherName>Springer</publisherName>
    <publisherPlace>Berlin</publisherPlace>
    <creatingCorporation/>
    <contributingCorporation/>
    <belongsToBibliography>0</belongsToBibliography>
    <completedDate>2022-02-22</completedDate>
    <publishedDate>--</publishedDate>
    <thesisDateAccepted>--</thesisDateAccepted>
    <title language="eng">The scenario coevolution paradigm</title>
    <abstract language="eng">Systems are becoming increasingly more adaptive, using techniques like machine learning to enhance their behavior on their own rather than only through human developers programming them. We analyze the impact the advent of these new techniques has on the discipline of rigorous software engineering, especially on the issue of quality assurance. To this end, we provide a general description of the processes related to machine learning and embed them into a formal framework for the analysis of adaptivity, recognizing that to test an adaptive system a new approach to adaptive testing is necessary. We introduce scenario coevolution as a design pattern describing how system and test can work as antagonists in the process of software evolution. While the general pattern applies to large-scale processes (including human developers further augmenting the system), we show all techniques on a smaller-scale example of an agent navigating a simple smart factory. We point out new aspects in software engineering for adaptive systems that may be tackled naturally using scenario coevolution. This work is a substantially extended take on Gabor et al. (International symposium on leveraging applications of formal methods, Springer, pp 137–154, 2018).</abstract>
    <parentTitle language="eng">International Journal on Software Tools for Technology Transfer</parentTitle>
    <subTitle language="deu">adaptive quality assurance for adaptive systems</subTitle>
    <identifier type="issn">1433-2787</identifier>
    <identifier type="urn">urn:nbn:de:bvb:573-13929</identifier>
    <enrichment key="THI_articleversion">published</enrichment>
    <enrichment key="THI_review">peer-review</enrichment>
    <enrichment key="THI_openaccess">ja</enrichment>
    <enrichment key="opus.source">publish</enrichment>
    <enrichment key="THI_relatedIdentifier">https://doi.org/10.1007/s10009-020-00560-5</enrichment>
    <licence>Creative Commons BY 4.0</licence>
    <author>
      <first_name>Thomas</first_name>
      <last_name>Gabor</last_name>
    </author>
    <author>
      <first_name>Andreas</first_name>
      <last_name>Sedlmeier</last_name>
    </author>
    <author>
      <first_name>Thomy</first_name>
      <last_name>Phan</last_name>
    </author>
    <author>
      <first_name>Fabian</first_name>
      <last_name>Ritz</last_name>
    </author>
    <author>
      <first_name>Marie</first_name>
      <last_name>Kiermeier</last_name>
    </author>
    <author>
      <first_name>Lenz</first_name>
      <last_name>Belzner</last_name>
    </author>
    <author>
      <first_name>Bernhard</first_name>
      <last_name>Kempter</last_name>
    </author>
    <author>
      <first_name>Cornel</first_name>
      <last_name>Klein</last_name>
    </author>
    <author>
      <first_name>Horst</first_name>
      <last_name>Sauer</last_name>
    </author>
    <author>
      <first_name>Reiner</first_name>
      <last_name>Schmid</last_name>
    </author>
    <author>
      <first_name>Jan</first_name>
      <last_name>Wieghardt</last_name>
    </author>
    <author>
      <first_name>Marc</first_name>
      <last_name>Zeller</last_name>
    </author>
    <author>
      <first_name>Claudia</first_name>
      <last_name>Linnhoff-Popien</last_name>
    </author>
    <collection role="open_access" number="">open_access</collection>
    <collection role="persons" number="43324">Belzner, Lenz</collection>
    <thesisPublisher>Technische Hochschule Ingolstadt</thesisPublisher>
    <file>https://opus4.kobv.de/opus4-haw/files/1392/Gabor2020_TheScenarioCoevolutionParadigm.pdf</file>
  </doc>
  <doc>
    <id>1393</id>
    <completedYear/>
    <publishedYear>2020</publishedYear>
    <thesisYearAccepted/>
    <language>eng</language>
    <pageFirst>456</pageFirst>
    <pageLast>461</pageLast>
    <pageNumber/>
    <edition/>
    <issue/>
    <volume/>
    <articleNumber/>
    <type>conferenceobject</type>
    <publisherName>ACM</publisherName>
    <publisherPlace>New York</publisherPlace>
    <creatingCorporation/>
    <contributingCorporation/>
    <belongsToBibliography>0</belongsToBibliography>
    <completedDate>2022-02-22</completedDate>
    <publishedDate>--</publishedDate>
    <thesisDateAccepted>--</thesisDateAccepted>
    <title language="eng">The Holy Grail of Quantum Artificial Intelligence: Major Challenges in Accelerating the Machine Learning Pipeline</title>
    <parentTitle language="eng">ICSEW'20 : Proceedings of the IEEE/ACM 42nd International Conference on Software Engineering Workshops</parentTitle>
    <identifier type="isbn">978-1-4503-7963-2</identifier>
    <enrichment key="THI_review">peer-review</enrichment>
    <enrichment key="THI_openaccess">nein</enrichment>
    <enrichment key="opus.source">publish</enrichment>
    <enrichment key="THI_relatedIdentifier">https://doi.org/10.1145/3387940.3391469</enrichment>
    <enrichment key="THI_conferenceName">ICSE '20: 42nd International Conference on Software Engineering, online, 27.06. - 19.07.2020</enrichment>
    <author>
      <first_name>Thomas</first_name>
      <last_name>Gabor</last_name>
    </author>
    <author>
      <first_name>Leo</first_name>
      <last_name>Sünkel</last_name>
    </author>
    <author>
      <first_name>Fabian</first_name>
      <last_name>Ritz</last_name>
    </author>
    <author>
      <first_name>Thomy</first_name>
      <last_name>Phan</last_name>
    </author>
    <author>
      <first_name>Lenz</first_name>
      <last_name>Belzner</last_name>
    </author>
    <author>
      <first_name>Christoph</first_name>
      <last_name>Roch</last_name>
    </author>
    <author>
      <first_name>Sebastian</first_name>
      <last_name>Feld</last_name>
    </author>
    <author>
      <first_name>Claudia</first_name>
      <last_name>Linnhoff-Popien</last_name>
    </author>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>quantum computing</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>artificial intelligence</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>software engineering</value>
    </subject>
    <collection role="persons" number="43324">Belzner, Lenz</collection>
  </doc>
  <doc>
    <id>1565</id>
    <completedYear/>
    <publishedYear>2018</publishedYear>
    <thesisYearAccepted/>
    <language>eng</language>
    <pageFirst>229</pageFirst>
    <pageLast>240</pageLast>
    <pageNumber/>
    <edition/>
    <issue/>
    <volume/>
    <articleNumber/>
    <type>conferenceobject</type>
    <publisherName>Springer</publisherName>
    <publisherPlace>Cham</publisherPlace>
    <creatingCorporation/>
    <contributingCorporation/>
    <belongsToBibliography>0</belongsToBibliography>
    <completedDate>2022-02-28</completedDate>
    <publishedDate>--</publishedDate>
    <thesisDateAccepted>--</thesisDateAccepted>
    <title language="eng">Risk-Sensitivity in Simulation Based Online Planning</title>
    <parentTitle language="eng">KI 2018: Advances in Artificial Intelligence: 41st German Conference on AI Proceedings</parentTitle>
    <identifier type="isbn">978-3-030-00111-7</identifier>
    <identifier type="isbn">978-3-030-00110-0</identifier>
    <enrichment key="THI_review">peer-review</enrichment>
    <enrichment key="THI_openaccess">nein</enrichment>
    <enrichment key="opus.source">publish</enrichment>
    <enrichment key="THI_relatedIdentifier">https://doi.org/10.1007/978-3-030-00111-7_20</enrichment>
    <enrichment key="THI_conferenceName">41st German Conference on AI, Berlin (Germany), 24.-28.09.2018</enrichment>
    <author>
      <first_name>Kyrill</first_name>
      <last_name>Schmid</last_name>
    </author>
    <editor>
      <first_name>Frank</first_name>
      <last_name>Trollmann</last_name>
    </editor>
    <author>
      <first_name>Lenz</first_name>
      <last_name>Belzner</last_name>
    </author>
    <editor>
      <first_name>Anni-Yasmin</first_name>
      <last_name>Turhan</last_name>
    </editor>
    <author>
      <first_name>Marie</first_name>
      <last_name>Kiermeier</last_name>
    </author>
    <author>
      <first_name>Alexander</first_name>
      <last_name>Neitz</last_name>
    </author>
    <author>
      <first_name>Thomy</first_name>
      <last_name>Phan</last_name>
    </author>
    <author>
      <first_name>Thomas</first_name>
      <last_name>Gabor</last_name>
    </author>
    <author>
      <first_name>Claudia</first_name>
      <last_name>Linnhoff-Popien</last_name>
    </author>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>online planning</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>risk-sensitivity</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>local planning</value>
    </subject>
    <collection role="persons" number="43324">Belzner, Lenz</collection>
  </doc>
  <doc>
    <id>1567</id>
    <completedYear/>
    <publishedYear>2018</publishedYear>
    <thesisYearAccepted/>
    <language>eng</language>
    <pageFirst>841</pageFirst>
    <pageLast>848</pageLast>
    <pageNumber/>
    <edition/>
    <issue/>
    <volume/>
    <articleNumber/>
    <type>conferenceobject</type>
    <publisherName>ACM</publisherName>
    <publisherPlace>New York</publisherPlace>
    <creatingCorporation/>
    <contributingCorporation/>
    <belongsToBibliography>0</belongsToBibliography>
    <completedDate>2022-02-28</completedDate>
    <publishedDate>--</publishedDate>
    <thesisDateAccepted>--</thesisDateAccepted>
    <title language="eng">Inheritance-based diversity measures for explicit convergence control in evolutionary algorithms</title>
    <parentTitle language="eng">GECCO '18: Proceedings of the 2018 Genetic and Evolutionary Computation Conference</parentTitle>
    <identifier type="isbn">978-1-4503-5618-3</identifier>
    <enrichment key="THI_review">peer-review</enrichment>
    <enrichment key="THI_openaccess">nein</enrichment>
    <enrichment key="opus.source">publish</enrichment>
    <enrichment key="THI_relatedIdentifier">https://doi.org/10.1145/3205455.3205630</enrichment>
    <enrichment key="THI_conferenceName">GECCO '18: Genetic and Evolutionary Computation Conference, Kyoto (Japan), 15.-19.07.2018</enrichment>
    <author>
      <first_name>Thomas</first_name>
      <last_name>Gabor</last_name>
    </author>
    <editor>
      <first_name>Hernan</first_name>
      <last_name>Aguirre</last_name>
    </editor>
    <author>
      <first_name>Lenz</first_name>
      <last_name>Belzner</last_name>
    </author>
    <author>
      <first_name>Claudia</first_name>
      <last_name>Linnhoff-Popien</last_name>
    </author>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>diversity</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>evolutionary algorithms</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>premature convergence</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>optimization</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>genetic drift</value>
    </subject>
    <collection role="persons" number="43324">Belzner, Lenz</collection>
  </doc>
  <doc>
    <id>3028</id>
    <completedYear/>
    <publishedYear>2022</publishedYear>
    <thesisYearAccepted/>
    <language>eng</language>
    <pageFirst>205</pageFirst>
    <pageLast>223</pageLast>
    <pageNumber/>
    <edition/>
    <issue>2</issue>
    <volume>28</volume>
    <articleNumber/>
    <type>article</type>
    <publisherName>MIT Press</publisherName>
    <publisherPlace>Cambridge</publisherPlace>
    <creatingCorporation/>
    <contributingCorporation/>
    <belongsToBibliography>1</belongsToBibliography>
    <completedDate>2022-12-19</completedDate>
    <publishedDate>--</publishedDate>
    <thesisDateAccepted>--</thesisDateAccepted>
    <title language="eng">Self-Replication in Neural Networks</title>
    <parentTitle language="eng">Artificial Life</parentTitle>
    <identifier type="issn">1530-9185</identifier>
    <enrichment key="THI_relatedIdentifier">https://doi.org/10.1162/artl_a_00359</enrichment>
    <enrichment key="THI_articleversion">published</enrichment>
    <enrichment key="THI_review">peer-review</enrichment>
    <enrichment key="THI_openaccess">nein</enrichment>
    <enrichment key="opus.source">publish</enrichment>
    <enrichment key="opus.doi.autoCreate">false</enrichment>
    <enrichment key="opus.urn.autoCreate">false</enrichment>
    <author>
      <first_name>Thomas</first_name>
      <last_name>Gabor</last_name>
    </author>
    <author>
      <first_name>Steffen</first_name>
      <last_name>Illium</last_name>
    </author>
    <author>
      <first_name>Maximilian</first_name>
      <last_name>Zorn</last_name>
    </author>
    <author>
      <first_name>Cristian</first_name>
      <last_name>Lenta</last_name>
    </author>
    <author>
      <first_name>Andy</first_name>
      <last_name>Mattausch</last_name>
    </author>
    <author>
      <first_name>Lenz</first_name>
      <last_name>Belzner</last_name>
    </author>
    <author>
      <first_name>Claudia</first_name>
      <last_name>Linnhoff-Popien</last_name>
    </author>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>neural network</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>self-replication</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>artificial chemistry system</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>soup</value>
    </subject>
    <subject>
      <language>eng</language>
      <type>uncontrolled</type>
      <value>weight space</value>
    </subject>
    <collection role="institutes" number="19311">Fakultät Elektro- und Informationstechnik</collection>
    <collection role="institutes" number="19379">AImotion Bavaria</collection>
    <collection role="persons" number="43324">Belzner, Lenz</collection>
  </doc>
  <doc>
    <id>4928</id>
    <completedYear/>
    <publishedYear>2024</publishedYear>
    <thesisYearAccepted/>
    <language>eng</language>
    <pageFirst/>
    <pageLast/>
    <pageNumber>36</pageNumber>
    <edition/>
    <issue>2</issue>
    <volume>38</volume>
    <articleNumber>34</articleNumber>
    <type>article</type>
    <publisherName>Springer</publisherName>
    <publisherPlace>Dordrecht</publisherPlace>
    <creatingCorporation/>
    <contributingCorporation/>
    <belongsToBibliography>1</belongsToBibliography>
    <completedDate>2024-07-22</completedDate>
    <publishedDate>--</publishedDate>
    <thesisDateAccepted>--</thesisDateAccepted>
    <title language="eng">Emergent cooperation from mutual acknowledgment exchange in multi-agent reinforcement learning</title>
    <abstract language="eng">Peer incentivization (PI) is a recent approach where all agents learn to reward or penalize each other in a distributed fashion, which often leads to emergent cooperation. Current PI mechanisms implicitly assume a flawless communication channel in order to exchange rewards. These rewards are directly incorporated into the learning process without any chance to respond with feedback. Furthermore, most PI approaches rely on global information, which limits scalability and applicability to real-world scenarios where only local information is accessible. In this paper, we propose Mutual Acknowledgment Token Exchange (MATE), a PI approach defined by a two-phase communication protocol to exchange acknowledgment tokens as incentives to shape individual rewards mutually. All agents condition their token transmissions on the locally estimated quality of their own situations based on environmental rewards and received tokens. MATE is completely decentralized and only requires local communication and information. We evaluate MATE in three social dilemma domains. Our results show that MATE is able to achieve and maintain significantly higher levels of cooperation than previous PI approaches. In addition, we evaluate the robustness of MATE in more realistic scenarios, where agents can deviate from the protocol and communication failures can occur. We also evaluate the sensitivity of MATE w.r.t. the choice of token values.</abstract>
    <parentTitle language="eng">Autonomous Agents and Multi-Agent Systems</parentTitle>
    <identifier type="issn">1573-7454</identifier>
    <identifier type="issn">1387-2532</identifier>
    <identifier type="urn">urn:nbn:de:bvb:573-49287</identifier>
    <enrichment key="opus_doi_flag">true</enrichment>
    <enrichment key="opus_import_data">{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"indexed":{"date-parts":[[2024,7,12]],"date-time":"2024-07-12T00:23:19Z","timestamp":1720743799151},"reference-count":69,"publisher":"Springer Science and Business Media LLC","issue":"2","license":[{"start":{"date-parts":[[2024,7,11]],"date-time":"2024-07-11T00:00:00Z","timestamp":1720656000000},"content-version":"tdm","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"},{"start":{"date-parts":[[2024,7,11]],"date-time":"2024-07-11T00:00:00Z","timestamp":1720656000000},"content-version":"vor","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0"}],"funder":[{"DOI":"10.13039\/501100005722","name":"Ludwig-Maximilians-Universit\u00e4t M\u00fcnchen","doi-asserted-by":"crossref"}],"content-domain":{"domain":["link.springer.com"],"crossmark-restriction":false},"short-container-title":["Auton Agent Multi-Agent Syst"],"published-print":{"date-parts":[[2024,12]]},"abstract":"&lt;jats:title&gt;Abstract&lt;\/jats:title&gt;&lt;jats:p&gt;&lt;jats:italic&gt;Peer incentivization (PI)&lt;\/jats:italic&gt; is a recent approach where all agents learn to reward or penalize each other in a distributed fashion, which often leads to emergent cooperation. Current PI mechanisms implicitly assume a flawless communication channel in order to exchange rewards. These rewards are directly incorporated into the learning process without any chance to respond with feedback. Furthermore, most PI approaches rely on global information, which limits scalability and applicability to real-world scenarios where only local information is accessible. In this paper, we propose &lt;jats:italic&gt;Mutual Acknowledgment Token Exchange (MATE)&lt;\/jats:italic&gt;, a PI approach defined by a two-phase communication protocol to exchange acknowledgment tokens as incentives to shape individual rewards mutually. All agents condition their token transmissions on the locally estimated quality of their own situations based on environmental rewards and received tokens. MATE is completely decentralized and only requires local communication and information. We evaluate MATE in three social dilemma domains. Our results show that MATE is able to achieve and maintain significantly higher levels of cooperation than previous PI approaches. In addition, we evaluate the robustness of MATE in more realistic scenarios, where agents can deviate from the protocol and communication failures can occur. We also evaluate the sensitivity of MATE w.r.t. the choice of token values.&lt;\/jats:p&gt;","DOI":"10.1007\/s10458-024-09666-5","type":"journal-article","created":{"date-parts":[[2024,7,11]],"date-time":"2024-07-11T06:01:45Z","timestamp":1720677705000},"update-policy":"http:\/\/dx.doi.org\/10.1007\/springer_crossmark_policy","source":"Crossref","is-referenced-by-count":0,"title":["Emergent cooperation from mutual acknowledgment exchange in multi-agent reinforcement learning"],"prefix":"10.1007","volume":"38","author":[{"given":"Thomy","family":"Phan","sequence":"first","affiliation":[]},{"given":"Felix","family":"Sommer","sequence":"additional","affiliation":[]},{"given":"Fabian","family":"Ritz","sequence":"additional","affiliation":[]},{"given":"Philipp","family":"Altmann","sequence":"additional","affiliation":[]},{"given":"Jonas","family":"N\u00fc\u00dflein","sequence":"additional","affiliation":[]},{"given":"Michael","family":"K\u00f6lle","sequence":"additional","affiliation":[]},{"given":"Lenz","family":"Belzner","sequence":"additional","affiliation":[]},{"given":"Claudia","family":"Linnhoff-Popien","sequence":"additional","affiliation":[]}],"member":"297","published-online":{"date-parts":[[2024,7,11]]},"reference":[{"issue":"5","key":"9666_CR1","doi-asserted-by":"publisher","first-page":"3897","DOI":"10.1007\/s10462-021-10097-x","volume":"55","author":"A Amirkhani","year":"2022","unstructured":"Amirkhani, A., &amp; Barshooi, A. H. (2022). Consensus in multi-agent systems: A review. Artificial Intelligence Review, 55(5), 3897\u20133935.","journal-title":"Artificial Intelligence Review"},{"key":"9666_CR2","volume-title":"The Evolution Of Cooperation","author":"R Axelrod","year":"1984","unstructured":"Axelrod, R. (1984). The Evolution Of Cooperation. New York: Basic Books."},{"issue":"4489","key":"9666_CR3","doi-asserted-by":"publisher","first-page":"1390","DOI":"10.1126\/science.7466396","volume":"211","author":"R Axelrod","year":"1981","unstructured":"Axelrod, R., &amp; Hamilton, W. D. (1981). The evolution of cooperation. Science, 211(4489), 1390\u20131396.","journal-title":"Science"},{"key":"9666_CR4","unstructured":"Babes, M., Munoz de Cote, E. &amp; Littman, M. L. (2008). Social reward shaping in the Prisoner\u2019s dilemma. In Proceedings of the 7th international joint conference on autonomous agents and multiagent systems-volume 3, pp. 1389\u20131392. International Foundation for Autonomous Agents and Multiagent Systems."},{"key":"9666_CR5","unstructured":"Barrett, S., Stone, P., &amp; Kraus, S. (2011). Empirical evaluation of Ad Hoc teamwork in the pursuit domain. In The 10th international conference on autonomous agents and multiagent systems - volume 2, AAMAS \u201911, pp. 567\u2013574. International Foundation for Autonomous Agents and Multiagent Systems."},{"issue":"2","key":"9666_CR6","doi-asserted-by":"publisher","first-page":"215","DOI":"10.1016\/S0004-3702(02)00121-2","volume":"136","author":"M Bowling","year":"2002","unstructured":"Bowling, M., &amp; Veloso, M. (2002). Multiagent learning using a variable learning rate. Artificial Intelligence, 136(2), 215\u2013250.","journal-title":"Artificial Intelligence"},{"issue":"2","key":"9666_CR7","doi-asserted-by":"publisher","first-page":"156","DOI":"10.1109\/TSMCC.2007.913919","volume":"38","author":"L Bu\u015foniu","year":"2008","unstructured":"Bu\u015foniu, L., Babu\u0161ka, R., &amp; De Schutter, B. (2008). Multi-agent reinforcement learning: An overview. IEEE Transactions on Systems, Man, and Cybernetics-Part C: Applications and Reviews, 38(2), 156\u2013172.","journal-title":"IEEE Transactions on Systems, Man, and Cybernetics-Part C: Applications and Reviews"},{"key":"9666_CR8","unstructured":"Christoffersen, P. J., Haupt, A. A., &amp; Hadfield-Menell, D. (2023). Get it in writing: Formal contracts mitigate social dilemmas in multi-agent RL. In 22nd international conference on autonomous agents and multiagent systems (AAMAS), AAMAS \u201923, pp. 448\u2013456. International Foundation for Autonomous Agents and Multiagent Systems."},{"issue":"8","key":"9666_CR9","doi-asserted-by":"publisher","first-page":"449","DOI":"10.1016\/j.tree.2005.05.008","volume":"20","author":"L Conradt","year":"2005","unstructured":"Conradt, L., &amp; Roper, T. J. (2005). Consensus decision making in animals. Trends in Ecology &amp; Evolution, 20(8), 449\u2013456.","journal-title":"Trends in Ecology &amp; Evolution"},{"key":"9666_CR10","series-title":"Oxford Landmark Science","volume-title":"The selfish gene: 40th (Anniversary)","author":"R Dawkins","year":"2016","unstructured":"Dawkins, R. (2016). The selfish gene: 40th (Anniversary). Oxford Landmark ScienceUK: OUP Oxford."},{"issue":"9","key":"9666_CR11","doi-asserted-by":"publisher","first-page":"6103","DOI":"10.1109\/TII.2020.2974875","volume":"16","author":"S Deng","year":"2020","unstructured":"Deng, S., Xiang, Z., Zhao, P., Taheri, J., Gao, H., Yin, J., &amp; Zomaya, A. Y. (2020). Dynamical resource allocation in edge for trustable internet-of-things systems: A reinforcement learning method. IEEE Transactions on Industrial Informatics, 16(9), 6103\u20136113.","journal-title":"IEEE Transactions on Industrial Informatics"},{"key":"9666_CR12","unstructured":"Devlin, S., &amp; Kudenko, D. (2011). Theoretical considerations of potential-based reward shaping for multi-agent systems. In The 10th international conference on autonomous agents and multiagent systems, pp. 225\u2013232. ACM, International Foundation for Autonomous Agents and Multiagent Systems."},{"key":"9666_CR13","unstructured":"Devlin, S., Yliniemi, L., Kudenko, D., &amp; Tumer, K. (2014). Potential-based difference rewards for multiagent reinforcement learning. In Proceedings of the 2014 international conference on autonomous agents and multi-agent systems, pp. 165\u2013172. International Foundation for Autonomous Agents and Multiagent Systems."},{"key":"9666_CR14","doi-asserted-by":"crossref","unstructured":"Dimeas, A. L., &amp; Hatziargyriou, N. D. (2010). Multi-agent reinforcement learning for microgrids. In IEEE PES General Meeting, pp. 1\u20138. IEEE.","DOI":"10.1109\/PES.2010.5589633"},{"key":"9666_CR15","unstructured":"Foerster, J., Assael, I. A., De Freitas, N., &amp; Whiteson, S. (2016). Learning to communicate with deep multi-agent reinforcement learning. In Advances in Neural Information Processing Systems, pp. 2137\u20132145. Red Hook: Curran Associates Inc."},{"key":"9666_CR16","unstructured":"Foerster, J. N., Chen, R. Y., Al-Shedivat, M., Whiteson, S., Abbeel, P., &amp; Mordatch, I. (2018). Learning with opponent-learning awareness. In Proceedings of the 17th international conference on autonomous agents and multiagent systems, pp. 122\u2013130. International Foundation for Autonomous Agents and Multiagent Systems."},{"key":"9666_CR17","doi-asserted-by":"publisher","first-page":"66","DOI":"10.1007\/978-3-319-71682-4_5","volume":"10642","author":"JK Gupta","year":"2017","unstructured":"Gupta, J. K., Egorov, M., &amp; Kochenderfer, M. (2017). Cooperative multi-agent control using deep reinforcement learning. Autonomous Agents and Multiagent Systems, 10642, 66\u201383.","journal-title":"Autonomous Agents and Multiagent Systems"},{"key":"9666_CR18","unstructured":"Guresti, B., Vanlioglu, A., &amp; Ure, N. K. (2023). IQ-flow: Mechanism design for inducing cooperative behavior to self-interested agents in sequential social dilemmas. In 22nd international conference on autonomous agents and multiagent systems (AAMAS), AAMAS \u201923, pp. 2143\u20132151. International Foundation for Autonomous Agents and Multiagent Systems."},{"key":"9666_CR19","doi-asserted-by":"crossref","unstructured":"Hahn, C., Phan, T., Gabor, T., Belzner, L., &amp; Linnhoff-Popien, C. (2019). Emergent escape-based flocking behavior using multi-agent reinforcement learning. volume ALIFE 2019: The 2019 Conference on Artificial Life of ALIFE 2019: The 2019 Conference on Artificial Life, pp. 598\u2013605. MIT Press.","DOI":"10.1162\/isal_a_00226.xml"},{"key":"9666_CR20","doi-asserted-by":"crossref","unstructured":"Hahn, C., Ritz, F., Wikidal, P., Phan, T., Gabor, T., &amp; Linnhoff-Popien, C. (2020). Foraging swarms using multi-agent reinforcement learning. volume ALIFE 2020: The 2020 Conference on Artificial Life of ALIFE 2020: The 2020 Conference on Artificial Life, pp. 333\u2013340. MIT Press.","DOI":"10.1162\/isal_a_00267"},{"key":"9666_CR21","unstructured":"Hennes, D., Morrill, D., Omidshafiei, S., Munos, R., Perolat, J., Lanctot, M., Gruslys, A., Lespiau, J.B., Parmas, P., Du\u00e9\u00f1ez-Guzm\u00e1n, E., &amp; Tuyls, K. (2020). Neural replicator dynamics: Multiagent learning via hedging policy gradients. In Proceedings of the 19th international conference on autonomous agents and multiagent systems, AAMAS \u201920, pp. 492\u2013501. International Foundation for Autonomous Agents and Multiagent Systems."},{"key":"9666_CR22","unstructured":"Hernandez-Leal, P., Kaisers, M., Baarslag, T., &amp; De Cote, E. M. (2017). A survey of learning in multiagent environments: Dealing with non-stationarity. arXiv preprint arXiv:1707.09183."},{"key":"9666_CR23","unstructured":"Hua, Y., Gao, S., Li, W., Jin, B., Wang, X., &amp; Zha, H. (2023). Learning optimal \"Pigovian Tax\" in sequential social dilemmas. In 22nd international conference on autonomous agents and multiagent systems (AAMAS), AAMAS \u201923, pp. 2784\u20132786. International Foundation for Autonomous Agents and Multiagent Systems."},{"key":"9666_CR24","unstructured":"Hughes, E., Leibo, J. Z., Phillips, M., Tuyls, K., Due\u00f1ez-Guzman, E., Garc\u00eda Casta\u00f1eda, A., Dunning, I., Zhu, T., McKee, K., Koster, R., et\u00a0al. (2018). Inequity aversion improves cooperation in intertemporal social dilemmas. In Proceedings of the 32nd international conference on neural information processing systems, pp. 3330\u20133340. Red Hook: Curran Associates Inc."},{"key":"9666_CR25","unstructured":"Ivanov, D., Zisman, I., &amp; Chernyshev, K. (2023). Mediated multi-agent reinforcement learning. In Proceedings of the 2023 international conference on autonomous agents and multi-agent systems, pp. 49\u201357. International Foundation for Autonomous Agents and Multiagent Systems."},{"issue":"6443","key":"9666_CR26","doi-asserted-by":"publisher","first-page":"859","DOI":"10.1126\/science.aau6249","volume":"364","author":"M Jaderberg","year":"2019","unstructured":"Jaderberg, M., Czarnecki, W. M., Dunning, I., Marris, L., Lever, G., Castaneda, A. G., Beattie, C., Rabinowitz, N. C., Morcos, A. S., Ruderman, A., et al. (2019). Human-level performance in 3D multiplayer games with population-based reinforcement learning. Science, 364(6443), 859\u2013865.","journal-title":"Science"},{"key":"9666_CR27","unstructured":"Jaques, N., Lazaridou, A., Hughes, E., Gulcehre, C., Ortega, P., Strouse, D. J., Leibo, J. Z., &amp; De Freitas, N. (2019). Social influence as intrinsic motivation for multi-agent deep reinforcement learning. In: Kamalika, C., &amp; Ruslan, S. (eds.) Proceedings of the 36th international conference on machine learning, volume\u00a097 of Proceedings of machine learning research, pp. 3040\u20133049. PMLR, 09\u201315."},{"key":"9666_CR28","unstructured":"K\u00f6ster, R., Hadfield-Menell, D., Hadfield, G. K., &amp; Leibo, J. Z. (2020). Silly rules improve the capacity of agents to learn stable enforcement and compliance behaviors. In Proceedings of the 19th international conference on autonomous agents and multiagent systems, AAMAS \u201920, pp. 1887\u20131888. International Foundation for Autonomous Agents and Multiagent Systems."},{"issue":"1","key":"9666_CR29","doi-asserted-by":"publisher","first-page":"55","DOI":"10.3233\/KES-2010-0206","volume":"15","author":"GJ Laurent","year":"2011","unstructured":"Laurent, G. J., Matignon, L., Fort-Piat, L., et al. (2011). The world of independent learners is not Markovian. International Journal of Knowledge-Based and Intelligent Engineering Systems, 15(1), 55\u201364.","journal-title":"International Journal of Knowledge-Based and Intelligent Engineering Systems"},{"key":"9666_CR30","unstructured":"Leibo, J. Z., Zambaldi, V., Lanctot, M., Marecki, J., &amp; Graepel, T. (2017). Multi-agent reinforcement learning in sequential social dilemmas. In Proceedings of the 16th conference on autonomous agents and multiagent systems, AAMAS \u201917, pp. 464\u2013473. International Foundation for Autonomous Agents and Multiagent Systems."},{"key":"9666_CR31","unstructured":"Lerer, A., &amp; Peysakhovich, A. (2017). Maintaining cooperation in complex social dilemmas using deep reinforcement learning. arXiv preprint arXiv:1707.01068."},{"key":"9666_CR32","unstructured":"Letcher, A., Foerster, J., Balduzzi, D., Rockt\u00e4schel, T., &amp; Whiteson, S. (2019). Stable opponent shaping in differentiable games. In International conference on learning representations."},{"key":"9666_CR33","doi-asserted-by":"crossref","unstructured":"Littman, M. L. (1994). Markov games as a framework for multi-agent reinforcement learning. In Machine learning proceedings 1994, pp. 157\u2013163. Morgan Kaufmann, San Francisco.","DOI":"10.1016\/B978-1-55860-335-6.50027-1"},{"key":"9666_CR34","unstructured":"Littman, M. L. (2001). Friend-or-foe Q-learning in general-sum games. In Proceedings of the eighteenth international conference on machine learning, ICML \u201901, pp. 322\u2013328. San Francisco: Morgan Kaufmann Publishers Inc."},{"key":"9666_CR35","unstructured":"Lowe, R., Wu, Y. I., Tamar, A., Harb, J., Pieter Abbeel, O., &amp; Mordatch, I. (2017). Multi-agent actor-critic for mixed cooperative-competitive environments. In Advances in Neural Information Processing Systems, vol. 30. Curran Associates, Inc."},{"key":"9666_CR36","unstructured":"Lupu, A., &amp; Precup, D. (2020). Gifting in multi-agent reinforcement learning. In Proceedings of the 19th international conference on autonomous agents and multiagent systems, pp. 789\u2013797. International Foundation for Autonomous Agents and Multiagent Systems."},{"key":"9666_CR37","doi-asserted-by":"crossref","unstructured":"Matignon, L., Laurent, G. J., &amp; Le Fort-Piat, N. (2007). Hysteretic Q-learning: An algorithm for decentralized reinforcement learning in cooperative multi-agent teams. In 2007 IEEE\/RSJ International Conference on Intelligent Robots and Systems, pp. 64\u201369. IEEE.","DOI":"10.1109\/IROS.2007.4399095"},{"key":"9666_CR38","unstructured":"M\u00fcller, R., Illium, S., Phan, T., Haider, T., &amp; Linnhoff-Popien, C. (2022). Towards anomaly detection in reinforcement learning. In 21st international conference on autonomous agents and multiagent systems (AAMAS), AAMAS \u201922, pp. 1799\u20131803. International Foundation for Autonomous Agents and Multiagent Systems."},{"key":"9666_CR39","doi-asserted-by":"crossref","unstructured":"Olfati-Saber, R., &amp; Shamma, J. S. (2005). Consensus filters for sensor networks and distributed sensor fusion. In Proceedings of the 44th IEEE conference on decision and control, pp. 6698\u20136703. IEEE.","DOI":"10.1109\/CDC.2005.1583238"},{"key":"9666_CR40","unstructured":"Orzan, N., Acar, E., Grossi, D., &amp; R\u0103dulescu, R. (2024). Emergent cooperation under uncertain incentive alignment. In 23rd international conference on autonomous agents and multiagent systems (AAMAS), AAMAS \u201924, pp. 1521\u20131530. International Foundation for Autonomous Agents and Multiagent Systems."},{"key":"9666_CR41","unstructured":"Perolat, J., Leibo, J. Z., Zambaldi, V., Beattie, C., Tuyls, K., &amp; Graepel, T. (2017). A multi-agent reinforcement learning model of common-pool resource appropriation. In: Proceedings of the 31st international conference on neural information processing systems, NIPS\u201917, pp. 3646\u20133655. Red Hook: Curran Associates Inc."},{"key":"9666_CR42","unstructured":"Peysakhovich, A., &amp; Lerer, A. (2018). Prosocial learning agents solve generalized stag hunts better than selfish ones. In Proceedings of the 17th international conference on autonomous agents and multiagent systems, AAMAS \u201918, pp. 2043\u20132044. International Foundation for Autonomous Agents and Multiagent Systems."},{"key":"9666_CR43","unstructured":"Phan, T., Ritz, F., Belzner, L., Altmann, P., Gabor, T., &amp; Linnhoff- Popien, C. (2021). VAST: Value function factorization with variable agent sub-teams. In Advances in neural information processing systems, pp. 24018\u201324032. Curran Associates Inc."},{"key":"9666_CR44","doi-asserted-by":"crossref","unstructured":"Phan, T., Sommer, F., Altmann, P., Ritz, F., Belzner, L., &amp; Linnhoff-Popien, C. (2022). Emergent cooperation from mutual acknowledgment exchange. In 21st international conference on autonomous agents and multiagent systems (AAMAS), AAMAS \u201922, pp. 1047\u20131055. International Foundation for Autonomous Agents and Multiagent Systems.","DOI":"10.21203\/rs.3.rs-2315844\/v1"},{"key":"9666_CR45","doi-asserted-by":"crossref","unstructured":"Radke, D., Larson, K., Brecht, T., &amp; Tilbury, K. (2023). Towards a better understanding of learning with multiagent teams. In Proceedings of the 32nd international joint conference on artificial intelligence, IJCAI-23, pp. 271\u2013279. International Joint Conferences on Artificial Intelligence Organization, 8.","DOI":"10.24963\/ijcai.2023\/31"},{"key":"9666_CR46","doi-asserted-by":"crossref","unstructured":"Rapoport, A. (1974). Prisoner\u2019s dilemma - recollections and observations. In Game theory as a theory of a conflict resolution, pp. 17\u201334. Springer.","DOI":"10.1007\/978-94-010-2161-6_2"},{"key":"9666_CR47","unstructured":"Rapoport, A., Chammah, A.\u00a0M., &amp; Orwant, C.\u00a0J. (1965). Prisoner\u2019s dilemma: A study in conflict and cooperation, volume 165. University of Michigan Press."},{"key":"9666_CR48","doi-asserted-by":"crossref","unstructured":"Ritz, F., Ratke, D., Phan, T., Belzner, L., &amp; Linnhoff-Popien, C. (2021). A sustainable ecosystem through emergent cooperation in multi-agent reinforcement learning. volume ALIFE 2021: The 2021 Conference on Artificial Life of ALIFE 2021: The 2021 Conference on Artificial Life. MIT Press, 07.","DOI":"10.1162\/isal_a_00399"},{"key":"9666_CR49","unstructured":"Roesch, S., Leonardos, S., &amp; Du, Y. (2024). The selfishness level of social dilemmas. In 23rd international conference on autonomous agents and multiagent systems (AAMAS), AAMAS \u201924, pp. 2441\u20132443. International Foundation for Autonomous Agents and Multiagent Systems."},{"key":"9666_CR50","doi-asserted-by":"crossref","unstructured":"Schenato, L., &amp; Gamba, G. (2007). A distributed consensus protocol for clock synchronization in wireless sensor network. Proceedings of the 46th IEEE conference on decision and control, pp. 2289\u20132294. IEEE.","DOI":"10.1109\/CDC.2007.4434671"},{"key":"9666_CR51","doi-asserted-by":"crossref","unstructured":"Schmid, K., Belzner, L., M\u00fcller, R., Tochtermann, J., &amp; Linnhoff-Popien, C. (2021). Stochastic market games. In: Zhi-Hua, Z. (ed.) Proceedings of the thirtieth international joint conference on artificial intelligence, IJCAI-21, pp. 384\u2013390. International Joint Conferences on Artificial Intelligence Organization, 8.","DOI":"10.24963\/ijcai.2021\/54"},{"key":"9666_CR52","doi-asserted-by":"crossref","unstructured":"Schmid, K., Belzner, L., Gabor, T., &amp; Phan, T. (2018). Action markets in deep multi-agent reinforcement learning. In Proceedings of the international conference on artificial neural networks, pp. 240\u2013249. Springer International Publishing.","DOI":"10.1007\/978-3-030-01421-6_24"},{"key":"9666_CR53","unstructured":"Shalev-Shwartz, S., Shammah, S., &amp; Shashua, A. (2016). Safe multi-agent reinforcement learning for autonomous driving. arXiv preprint arXiv:1610.03295."},{"key":"9666_CR54","doi-asserted-by":"publisher","first-page":"103535","DOI":"10.1016\/j.artint.2021.103535","volume":"299","author":"D Silver","year":"2021","unstructured":"Silver, D., Singh, S., Precup, D., &amp; Sutton, R. S. (2021). Reward is enough. Artificial Intelligence, 299, 103535.","journal-title":"Artificial Intelligence"},{"key":"9666_CR55","doi-asserted-by":"crossref","unstructured":"Speranzon, A., Fischione, C., &amp; Johansson, K. H. (2006). Distributed and collaborative estimation over wireless sensor networks. In: Proceedings of the 45th IEEE conference on decision and control, pp. 1025\u20131030. IEEE.","DOI":"10.1109\/CDC.2006.377101"},{"key":"9666_CR56","doi-asserted-by":"crossref","unstructured":"Stone, P., Kaminka, G., Kraus, S., &amp; Rosenschein, J. (2010). Ad hoc autonomous agent teams: collaboration without pre-coordination. In Proceedings of the AAAI conference on artificial intelligence, 24(1), 1504\u20131509.","DOI":"10.1609\/aaai.v24i1.7529"},{"issue":"1","key":"9666_CR57","doi-asserted-by":"publisher","first-page":"9","DOI":"10.1007\/BF00115009","volume":"3","author":"RS Sutton","year":"1988","unstructured":"Sutton, R. S. (1988). Learning to predict by the methods of temporal differences. Machine Learning, 3(1), 9\u201344.","journal-title":"Machine Learning"},{"key":"9666_CR58","volume-title":"Reinforcement learning: An introduction","author":"RS Sutton","year":"2018","unstructured":"Sutton, R. S., &amp; Barto, A. G. (2018). Reinforcement learning: An introduction. MIT Press."},{"key":"9666_CR59","unstructured":"Sutton, R.\u00a0S., McAllester, D.\u00a0A., Singh, S.\u00a0P., &amp; Mansour, Y. (2000). Policy gradient methods for reinforcement learning with function approximation. In S.\u00a0Solla, T.\u00a0Leen, and K.\u00a0M\u00fcller, (Eds.), Advances in neural information processing systems, vol.\u00a012, pp. 1057\u20131063. MIT Press."},{"key":"9666_CR60","doi-asserted-by":"crossref","unstructured":"Tan, M. (1993). Multi-agent reinforcement learning: Independent versus cooperative agents. In Proceedings of the tenth international conference on international conference on machine learning, pp. 330\u2013337. Morgan Kaufmann Publishers Inc.","DOI":"10.1016\/B978-1-55860-307-3.50049-6"},{"key":"9666_CR61","unstructured":"Andrew, S. (2007). Tanenbaum and Maarten Van Steen. Distributed Systems: Principles and Paradigms. Prentice-Hall."},{"issue":"1","key":"9666_CR62","doi-asserted-by":"publisher","first-page":"35","DOI":"10.1086\/406755","volume":"46","author":"RL Trivers","year":"1971","unstructured":"Trivers, R. L. (1971). The evolution of reciprocal altruism. The Quarterly Review of Biology, 46(1), 35\u201357.","journal-title":"The Quarterly Review of Biology"},{"issue":"2","key":"9666_CR63","doi-asserted-by":"publisher","first-page":"125","DOI":"10.1016\/j.obhdp.2012.11.003","volume":"120","author":"PAM Van Lange","year":"2013","unstructured":"Van Lange, P. A. M., Joireman, J., Parks, C. D., &amp; Van Dijk, E. (2013). The psychology of social dilemmas: A review. Organizational Behavior and Human Decision Processes, 120(2), 125\u2013141.","journal-title":"Organizational Behavior and Human Decision Processes"},{"key":"9666_CR64","unstructured":"Vinitsky, E., K\u00f6ster, R., Agapiou, J.\u00a0P., Du\u00e9\u00f1ez-Guzm\u00e1n, E., Vezhnevets, A.\u00a0S., &amp; Leibo, J.\u00a0Z. (2021). A learning agent that acquires social norms from public sanctions in decentralized multi-agent settings. arXiv preprint arXiv:2106.09012."},{"issue":"7782","key":"9666_CR65","doi-asserted-by":"publisher","first-page":"350","DOI":"10.1038\/s41586-019-1724-z","volume":"575","author":"O Vinyals","year":"2019","unstructured":"Vinyals, O., Babuschkin, I., Czarnecki, W. M., Mathieu, M., Dudzik, A., Chung, J., Choi, D. H., Powell, R., Ewalds, T., Georgiev, P., et al. (2019). Grandmaster level in starcraft ii using multi-agent reinforcement learning. Nature, 575(7782), 350\u2013354.","journal-title":"Nature"},{"issue":"1","key":"9666_CR66","first-page":"2914","volume":"17","author":"E Wei","year":"2016","unstructured":"Wei, E., &amp; Luke, S. (2016). Lenient learning in independent-learner stochastic cooperative games. The Journal of Machine Learning Research, 17(1), 2914\u20132955.","journal-title":"The Journal of Machine Learning Research"},{"issue":"3","key":"9666_CR67","doi-asserted-by":"publisher","first-page":"229","DOI":"10.1007\/BF00992696","volume":"8","author":"RJ Williams","year":"1992","unstructured":"Williams, R. J. (1992). Simple statistical gradient-following algorithms for connectionist reinforcement learning. Machine Learning, 8(3), 229\u2013256.","journal-title":"Machine Learning"},{"key":"9666_CR68","doi-asserted-by":"crossref","unstructured":"Yang, J., Li, A., Farajtabar, M., Sunehag, P., Hughes, E., &amp; Zha, H. (2020). Learning to incentivize other learning agents. Advances in Neural Information Processing Systems, 33.","DOI":"10.1007\/978-3-030-63823-8"},{"key":"9666_CR69","unstructured":"Yang, Y., Luo, R., Li, M., Zhou, M., Zhang, W., &amp; Wang, J. (2018). Mean field multi-agent reinforcement learning. In 35th international conference on machine learning, ICML 2018, vol.\u00a080, pp. 5571\u20135580. PMLR."}],"container-title":["Autonomous Agents and Multi-Agent Systems"],"original-title":[],"language":"en","link":[{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10458-024-09666-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/article\/10.1007\/s10458-024-09666-5\/fulltext.html","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/link.springer.com\/content\/pdf\/10.1007\/s10458-024-09666-5.pdf","content-type":"application\/pdf","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2024,7,11]],"date-time":"2024-07-11T06:16:42Z","timestamp":1720678602000},"score":1,"resource":{"primary":{"URL":"https:\/\/link.springer.com\/10.1007\/s10458-024-09666-5"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2024,7,11]]},"references-count":69,"journal-issue":{"issue":"2","published-print":{"date-parts":[[2024,12]]}},"alternative-id":["9666"],"URL":"http:\/\/dx.doi.org\/10.1007\/s10458-024-09666-5","relation":{},"ISSN":["1387-2532","1573-7454"],"issn-type":[{"value":"1387-2532","type":"print"},{"value":"1573-7454","type":"electronic"}],"subject":[],"published":{"date-parts":[[2024,7,11]]},"assertion":[{"value":"2 July 2024","order":1,"name":"accepted","label":"Accepted","group":{"name":"ArticleHistory","label":"Article History"}},{"value":"11 July 2024","order":2,"name":"first_online","label":"First Online","group":{"name":"ArticleHistory","label":"Article History"}},{"order":1,"name":"Ethics","group":{"name":"EthicsHeading","label":"Declarations"}},{"value":"T.P. contributed to the work at LMU Munich and is now affiliated with the University of Southern California. F.S., F.R., P.A., J.N., M.K., and C.L. contributed to the work while affiliated with LMU Munich. L.B. contributed to the work while affiliated with Technische Hochschule Ingolstadt.","order":2,"name":"Ethics","group":{"name":"EthicsHeading","label":"Conflict of interest"}}],"article-number":"34"}}</enrichment>
    <enrichment key="local_crossrefDocumentType">journal-article</enrichment>
    <enrichment key="local_crossrefLicence">https://creativecommons.org/licenses/by/4.0</enrichment>
    <enrichment key="local_import_origin">crossref</enrichment>
    <enrichment key="local_doiImportPopulated">PersonAuthorFirstName_1,PersonAuthorLastName_1,PersonAuthorFirstName_2,PersonAuthorLastName_2,PersonAuthorFirstName_3,PersonAuthorLastName_3,PersonAuthorFirstName_4,PersonAuthorLastName_4,PersonAuthorFirstName_5,PersonAuthorLastName_5,PersonAuthorFirstName_6,PersonAuthorLastName_6,PersonAuthorFirstName_7,PersonAuthorLastName_7,PersonAuthorFirstName_8,PersonAuthorLastName_8,PublisherName,TitleMain_1,Language,TitleAbstract_1,TitleParent_1,ArticleNumber,Issue,Volume,CompletedYear,IdentifierIssn,Enrichmentlocal_crossrefLicence</enrichment>
    <enrichment key="opus.source">doi-import</enrichment>
    <enrichment key="THI_relatedIdentifier">https://doi.org/10.1007/s10458-024-09666-5</enrichment>
    <enrichment key="THI_openaccess">ja</enrichment>
    <enrichment key="THI_review">peer-review</enrichment>
    <enrichment key="THI_articleversion">published</enrichment>
    <enrichment key="THI_furtherversion">https://opus4.kobv.de/opus4-haw/frontdoor/index/index/docId/4547</enrichment>
    <licence>Creative Commons BY 4.0</licence>
    <author>
      <first_name>Thomy</first_name>
      <last_name>Phan</last_name>
    </author>
    <author>
      <first_name>Felix</first_name>
      <last_name>Sommer</last_name>
    </author>
    <author>
      <first_name>Fabian</first_name>
      <last_name>Ritz</last_name>
    </author>
    <author>
      <first_name>Philipp</first_name>
      <last_name>Altmann</last_name>
    </author>
    <author>
      <first_name>Jonas</first_name>
      <last_name>Nüßlein</last_name>
    </author>
    <author>
      <first_name>Michael</first_name>
      <last_name>Kölle</last_name>
    </author>
    <author>
      <first_name>Lenz</first_name>
      <last_name>Belzner</last_name>
    </author>
    <author>
      <first_name>Claudia</first_name>
      <last_name>Linnhoff-Popien</last_name>
    </author>
    <collection role="open_access" number="">open_access</collection>
    <collection role="institutes" number="19311">Fakultät Elektro- und Informationstechnik</collection>
    <collection role="institutes" number="19379">AImotion Bavaria</collection>
    <collection role="persons" number="43324">Belzner, Lenz</collection>
    <thesisPublisher>Technische Hochschule Ingolstadt</thesisPublisher>
    <file>https://opus4.kobv.de/opus4-haw/files/4928/s10458-024-09666-5.pdf</file>
  </doc>
  <doc>
    <id>4547</id>
    <completedYear/>
    <publishedYear>2022</publishedYear>
    <thesisYearAccepted/>
    <language>eng</language>
    <pageFirst/>
    <pageLast/>
    <pageNumber>52</pageNumber>
    <edition/>
    <issue/>
    <volume/>
    <articleNumber/>
    <type>preprint</type>
    <publisherName>Research Square</publisherName>
    <publisherPlace>Durham</publisherPlace>
    <creatingCorporation/>
    <contributingCorporation/>
    <belongsToBibliography>0</belongsToBibliography>
    <completedDate>--</completedDate>
    <publishedDate>--</publishedDate>
    <thesisDateAccepted>--</thesisDateAccepted>
    <title language="eng">Emergent Cooperation from Mutual Acknowledgment Exchange in Multi-Agent Reinforcement Learning</title>
    <abstract language="eng">Peer incentivization (PI) is a recent approach, where all agents learn to reward or to penalize each other in a distributed fashion which often leads to emergent cooperation. Current PI mechanisms implicitly assume a flawless communication channel in order to exchange rewards. These rewards are directly integrated into the learning process without any chance to respond with feedback. Furthermore, most PI approaches rely on global information which limits scalability and applicability to real-world scenarios, where only local information is accessible. In this paper, we propose Mutual Acknowledgment Token Exchange (MATE), a PI approach defined by a two-phase communication protocol to mutually exchange acknowledgment tokens to shape individual rewards. Each agent evaluates the monotonic improvement of its individual situation in order to accept or reject acknowledgment requests from other agents. MATE is completely decentralized and only requires local communication and information. We evaluate MATE in three social dilemma domains. Our results show that MATE is able to achieve and maintain significantly higher levels of cooperation than previous PI approaches. In addition, we evaluate the robustness of MATE in more realistic scenarios, where agents can defect from the protocol and where communication failures can occur. We also evaluate the sensitivity of MATE w.r.t. the choice of token values.</abstract>
    <parentTitle language="eng">Research Square</parentTitle>
    <identifier type="issn">2693-5015</identifier>
    <identifier type="urn">urn:nbn:de:bvb:573-45471</identifier>
    <enrichment key="opus_doi_flag">true</enrichment>
    <enrichment key="opus_import_data">{"status":"ok","message-type":"work","message-version":"1.0.0","message":{"institution":[{"name":"Research Square"}],"indexed":{"date-parts":[[2022,12,2]],"date-time":"2022-12-02T05:53:47Z","timestamp":1669960427813},"posted":{"date-parts":[[2022,12,1]]},"group-title":"In Review","reference-count":0,"publisher":"Research Square Platform LLC","license":[{"start":{"date-parts":[[2022,12,1]],"date-time":"2022-12-01T00:00:00Z","timestamp":1669852800000},"content-version":"unspecified","delay-in-days":0,"URL":"https:\/\/creativecommons.org\/licenses\/by\/4.0\/"}],"content-domain":{"domain":[],"crossmark-restriction":false},"short-container-title":[],"accepted":{"date-parts":[[2022,11,26]]},"abstract":"&lt;jats:title&gt;Abstract&lt;\/jats:title&gt;\n        &lt;jats:p&gt;Peer incentivization (PI) is a recent approach, where all agents learn to reward or to penalize each other in a distributed fashion which often leads to emergent cooperation. Current PI mechanisms implicitly assume a flawless communication channel in order to exchange rewards. These rewards are directly integrated into the learning process without any chance to respond with feedback. Furthermore, most PI approaches rely on global information which limits scalability and applicability to real-world scenarios, where only local information is accessible. In this paper, we propose Mutual Acknowledgment Token Exchange (MATE), a PI approach defined by a two-phase communication protocol to mutually exchange acknowledgment tokens to shape individual rewards. Each agent evaluates the monotonic improvement of its individual situation in order to accept or reject acknowledgment requests from other agents. MATE is completely decentralized and only requires local communication and information. We evaluate MATE in three social dilemma domains. Our results show that MATE is able to achieve and maintain significantly higher levels of cooperation than previous PI approaches. In addition, we evaluate the robustness of MATE in more realistic scenarios, where agents can defect from the protocol and where communication failures can occur. We also evaluate the sensitivity of MATE w.r.t. the choice of token values.&lt;\/jats:p&gt;","DOI":"10.21203\/rs.3.rs-2315844\/v1","type":"posted-content","created":{"date-parts":[[2022,12,1]],"date-time":"2022-12-01T20:23:29Z","timestamp":1669926209000},"source":"Crossref","is-referenced-by-count":0,"title":["Emergent Cooperation from Mutual Acknowledgment Exchange in Multi-Agent Reinforcement Learning"],"prefix":"10.21203","author":[{"given":"Thomy","family":"Phan","sequence":"first","affiliation":[{"name":"LMU Munich"}]},{"given":"Felix","family":"Sommer","sequence":"additional","affiliation":[{"name":"LMU Munich"}]},{"given":"Fabian","family":"Ritz","sequence":"additional","affiliation":[{"name":"LMU Munich"}]},{"given":"Philipp","family":"Altmann","sequence":"additional","affiliation":[{"name":"LMU Munich"}]},{"given":"Jonas","family":"N\u00fc\u00dflein","sequence":"additional","affiliation":[{"name":"LMU Munich"}]},{"given":"Michael","family":"K\u00f6lle","sequence":"additional","affiliation":[{"name":"LMU Munich"}]},{"given":"Lenz","family":"Belzner","sequence":"additional","affiliation":[{"name":"Technische Hochschule Ingolstadt"}]},{"given":"Claudia","family":"Linnhoff-Popien","sequence":"additional","affiliation":[{"name":"LMU Munich"}]}],"member":"8761","container-title":[],"original-title":[],"link":[{"URL":"https:\/\/www.researchsquare.com\/article\/rs-2315844\/v1","content-type":"text\/html","content-version":"vor","intended-application":"text-mining"},{"URL":"https:\/\/www.researchsquare.com\/article\/rs-2315844\/v1.html","content-type":"unspecified","content-version":"vor","intended-application":"similarity-checking"}],"deposited":{"date-parts":[[2022,12,1]],"date-time":"2022-12-01T20:23:30Z","timestamp":1669926210000},"score":1,"resource":{"primary":{"URL":"https:\/\/www.researchsquare.com\/article\/rs-2315844\/v1"}},"subtitle":[],"short-title":[],"issued":{"date-parts":[[2022,12,1]]},"references-count":0,"URL":"http:\/\/dx.doi.org\/10.21203\/rs.3.rs-2315844\/v1","relation":{},"published":{"date-parts":[[2022,12,1]]},"subtype":"preprint"}}</enrichment>
    <enrichment key="local_crossrefDocumentType">posted-content/preprint</enrichment>
    <enrichment key="local_crossrefLicence">https://creativecommons.org/licenses/by/4.0/</enrichment>
    <enrichment key="local_import_origin">crossref</enrichment>
    <enrichment key="local_doiImportPopulated">PersonAuthorFirstName_1,PersonAuthorLastName_1,PersonAuthorFirstName_2,PersonAuthorLastName_2,PersonAuthorFirstName_3,PersonAuthorLastName_3,PersonAuthorFirstName_4,PersonAuthorLastName_4,PersonAuthorFirstName_5,PersonAuthorLastName_5,PersonAuthorFirstName_6,PersonAuthorLastName_6,PersonAuthorFirstName_7,PersonAuthorLastName_7,PersonAuthorFirstName_8,PersonAuthorLastName_8,PublisherName,TitleMain_1,TitleAbstract_1,CompletedYear,Enrichmentlocal_crossrefLicence</enrichment>
    <enrichment key="opus.source">doi-import</enrichment>
    <enrichment key="THI_relatedIdentifier">https://doi.org/10.21203/rs.3.rs-2315844/v1</enrichment>
    <enrichment key="THI_openaccess">ja</enrichment>
    <enrichment key="THI_review">nein</enrichment>
    <enrichment key="THI_furtherversion">https://opus4.kobv.de/opus4-haw/frontdoor/index/index/docId/4928</enrichment>
    <licence>Creative Commons BY 4.0</licence>
    <author>
      <first_name>Thomy</first_name>
      <last_name>Phan</last_name>
    </author>
    <author>
      <first_name>Felix</first_name>
      <last_name>Sommer</last_name>
    </author>
    <author>
      <first_name>Fabian</first_name>
      <last_name>Ritz</last_name>
    </author>
    <author>
      <first_name>Philipp</first_name>
      <last_name>Altmann</last_name>
    </author>
    <author>
      <first_name>Jonas</first_name>
      <last_name>Nüßlein</last_name>
    </author>
    <author>
      <first_name>Michael</first_name>
      <last_name>Kölle</last_name>
    </author>
    <author>
      <first_name>Lenz</first_name>
      <last_name>Belzner</last_name>
    </author>
    <author>
      <first_name>Claudia</first_name>
      <last_name>Linnhoff-Popien</last_name>
    </author>
    <collection role="open_access" number="">open_access</collection>
    <collection role="institutes" number="19311">Fakultät Elektro- und Informationstechnik</collection>
    <collection role="institutes" number="19379">AImotion Bavaria</collection>
    <collection role="persons" number="43324">Belzner, Lenz</collection>
    <thesisPublisher>Technische Hochschule Ingolstadt</thesisPublisher>
    <file>https://opus4.kobv.de/opus4-haw/files/4547/94bc3a0c635806ddee01bc47.pdf</file>
  </doc>
  <doc>
    <id>1395</id>
    <completedYear/>
    <publishedYear>2020</publishedYear>
    <thesisYearAccepted/>
    <language>eng</language>
    <pageFirst>522</pageFirst>
    <pageLast>529</pageLast>
    <pageNumber/>
    <edition/>
    <issue/>
    <volume/>
    <articleNumber/>
    <type>conferenceobject</type>
    <publisherName>SciTePress</publisherName>
    <publisherPlace>Setúbal</publisherPlace>
    <creatingCorporation/>
    <contributingCorporation/>
    <belongsToBibliography>0</belongsToBibliography>
    <completedDate>2022-02-22</completedDate>
    <publishedDate>--</publishedDate>
    <thesisDateAccepted>--</thesisDateAccepted>
    <title language="eng">Uncertainty-based out-of-distribution classification in deep reinforcement learning</title>
    <abstract language="eng">Robustness to out-of-distribution (OOD) data is an important goal in building reliable machine learning systems. As a first step towards a solution, we consider the problem of detecting such data in a value-based deep reinforcement learning (RL) setting. Modelling this problem as a one-class classification problem, we propose a framework for uncertainty-based OOD classification: UBOOD. It is based on the effect that an agent’s epistemic uncertainty is reduced for situations encountered during training (in-distribution), and thus lower than for unencountered (OOD) situations. Being agnostic towards the approach used for estimating epistemic uncertainty, combinations with different uncertainty estimation methods, e.g. approximate Bayesian inference methods or ensembling techniques are possible. Evaluation shows that the framework produces reliable classification results when combined with ensemble-based estimators, while the combination with concrete dropout-based estimators fails to r eliably detect OOD situations.</abstract>
    <parentTitle language="eng">Proceedings of the 12th International Conference on Agents and Artificial Intelligence - Volume 2: ICAART</parentTitle>
    <identifier type="isbn">978-989-758-395-7</identifier>
    <identifier type="issn">2184-433X</identifier>
    <identifier type="urn">urn:nbn:de:bvb:573-13959</identifier>
    <enrichment key="THI_review">peer-review</enrichment>
    <enrichment key="opus.source">publish</enrichment>
    <enrichment key="THI_relatedIdentifier">https://doi.org/10.5220/0008949905220529</enrichment>
    <enrichment key="THI_conferenceName">12th International Conference on Agents and Artificial Intelligence, Valletta (Malta), 22.-24.02.2020</enrichment>
    <licence>Creative Commons BY-NC-ND 4.0</licence>
    <author>
      <first_name>Andreas</first_name>
      <last_name>Sedlmeier</last_name>
    </author>
    <editor>
      <first_name>Ana Paula</first_name>
      <last_name>Rocha</last_name>
    </editor>
    <author>
      <first_name>Thomas</first_name>
      <last_name>Gabor</last_name>
    </author>
    <editor>
      <first_name>Luc</first_name>
      <last_name>Steels</last_name>
    </editor>
    <author>
      <first_name>Thomy</first_name>
      <last_name>Phan</last_name>
    </author>
    <editor>
      <first_name>Jaap van den</first_name>
      <last_name>Herik</last_name>
    </editor>
    <author>
      <first_name>Lenz</first_name>
      <last_name>Belzner</last_name>
    </author>
    <author>
      <first_name>Claudia</first_name>
      <last_name>Linnhoff-Popien</last_name>
    </author>
    <subject>
      <language>deu</language>
      <type>uncontrolled</type>
      <value>Uncertainty in AI</value>
    </subject>
    <subject>
      <language>deu</language>
      <type>uncontrolled</type>
      <value>Out-of-Distribution Classification</value>
    </subject>
    <subject>
      <language>deu</language>
      <type>uncontrolled</type>
      <value>Deep Reinforcement Learning</value>
    </subject>
    <collection role="open_access" number="">open_access</collection>
    <collection role="persons" number="43324">Belzner, Lenz</collection>
    <thesisPublisher>Technische Hochschule Ingolstadt</thesisPublisher>
    <file>https://opus4.kobv.de/opus4-haw/files/1395/89499.pdf</file>
  </doc>
  <doc>
    <id>1394</id>
    <completedYear/>
    <publishedYear>2020</publishedYear>
    <thesisYearAccepted/>
    <language>eng</language>
    <pageFirst>144</pageFirst>
    <pageLast>151</pageLast>
    <pageNumber/>
    <edition/>
    <issue/>
    <volume/>
    <articleNumber/>
    <type>conferenceobject</type>
    <publisherName>SciTePress</publisherName>
    <publisherPlace>Setúbal</publisherPlace>
    <creatingCorporation/>
    <contributingCorporation/>
    <belongsToBibliography>0</belongsToBibliography>
    <completedDate>2022-02-22</completedDate>
    <publishedDate>--</publishedDate>
    <thesisDateAccepted>--</thesisDateAccepted>
    <title language="eng">Multi-agent reinforcement learning for bargaining under risk and asymmetric information</title>
    <abstract language="deu">In cooperative game theory bargaining games refer to situations where players can agree to any one of a variety of outcomes but there is a conflict on which specific outcome to choose. However, the players cannot impose a specific outcome on others and if no agreement is reached all players receive a predetermined status quo outcome. Bargaining games have been studied from a variety of fields, including game theory, economics, psychology and simulation based methods like genetic algorithms. In this work we extend the analysis by means of deep multi-agent reinforcement learning (MARL). To study the dynamics of bargaining with reinforcement learning we propose two different bargaining environments which display the following situations: in the first domain two agents have to agree on the division of an asset, e.g., the division of a fixed amount of money between each other. The second domain models a seller-buyer scenario in which agents must agree on a price for a product. We empirica lly demonstrate that the bargaining result under MARL is influenced by agents’ risk-aversion as well as information asymmetry between agents.</abstract>
    <parentTitle language="eng">Proceedings of the 12th International Conference on Agents and Artificial Intelligence - Volume 1: ICAART 2020</parentTitle>
    <identifier type="isbn">978-989-758-395-7</identifier>
    <identifier type="issn">2184-433X</identifier>
    <identifier type="urn">urn:nbn:de:bvb:573-13945</identifier>
    <enrichment key="THI_review">peer-review</enrichment>
    <enrichment key="opus.source">publish</enrichment>
    <enrichment key="THI_relatedIdentifier">https://doi.org/10.5220/0008913901440151</enrichment>
    <enrichment key="THI_conferenceName">12th International Conference on Agents and Artificial Intelligence, Valletta (Malta), 22.-24.02.2020</enrichment>
    <enrichment key="THI_openaccess">ja</enrichment>
    <licence>Creative Commons BY-NC-ND 4.0</licence>
    <author>
      <first_name>Kyrill</first_name>
      <last_name>Schmid</last_name>
    </author>
    <editor>
      <first_name>Ana Paula</first_name>
      <last_name>Rocha</last_name>
    </editor>
    <author>
      <first_name>Lenz</first_name>
      <last_name>Belzner</last_name>
    </author>
    <editor>
      <first_name>Luc</first_name>
      <last_name>Steels</last_name>
    </editor>
    <author>
      <first_name>Thomy</first_name>
      <last_name>Phan</last_name>
    </author>
    <editor>
      <first_name>Jaap van den</first_name>
      <last_name>Herik</last_name>
    </editor>
    <author>
      <first_name>Thomas</first_name>
      <last_name>Gabor</last_name>
    </author>
    <author>
      <first_name>Claudia</first_name>
      <last_name>Linnhoff-Popien</last_name>
    </author>
    <collection role="open_access" number="">open_access</collection>
    <collection role="persons" number="43324">Belzner, Lenz</collection>
    <thesisPublisher>Technische Hochschule Ingolstadt</thesisPublisher>
    <file>https://opus4.kobv.de/opus4-haw/files/1394/89139.pdf</file>
  </doc>
  <doc>
    <id>1389</id>
    <completedYear/>
    <publishedYear>2021</publishedYear>
    <thesisYearAccepted/>
    <language>eng</language>
    <pageFirst>384</pageFirst>
    <pageLast>390</pageLast>
    <pageNumber/>
    <edition/>
    <issue/>
    <volume/>
    <articleNumber/>
    <type>conferenceobject</type>
    <publisherName>IJCAI</publisherName>
    <publisherPlace>[s. l.]</publisherPlace>
    <creatingCorporation/>
    <contributingCorporation/>
    <belongsToBibliography>1</belongsToBibliography>
    <completedDate>2022-02-22</completedDate>
    <publishedDate>--</publishedDate>
    <thesisDateAccepted>--</thesisDateAccepted>
    <title language="eng">Stochastic market games</title>
    <parentTitle language="eng">Proceedings of the Thirtieth International Joint Conference on Artificial Intelligence</parentTitle>
    <identifier type="isbn">978-0-9992411-9-6</identifier>
    <enrichment key="THI_review">peer-review</enrichment>
    <enrichment key="THI_openaccess">nein</enrichment>
    <enrichment key="opus.source">publish</enrichment>
    <enrichment key="THI_relatedIdentifier">https://doi.org/10.24963/ijcai.2021/54</enrichment>
    <enrichment key="THI_conferenceName">Thirtieth International Joint Conference on Artificial Intelligence, online, 19.-27.08.2021</enrichment>
    <author>
      <first_name>Kyrill</first_name>
      <last_name>Schmid</last_name>
    </author>
    <author>
      <first_name>Lenz</first_name>
      <last_name>Belzner</last_name>
    </author>
    <author>
      <first_name>Robert</first_name>
      <last_name>Müller</last_name>
    </author>
    <author>
      <first_name>Johannes</first_name>
      <last_name>Tochtermann</last_name>
    </author>
    <author>
      <first_name>Claudia</first_name>
      <last_name>Linnhoff-Popien</last_name>
    </author>
    <collection role="institutes" number="19311">Fakultät Elektro- und Informationstechnik</collection>
    <collection role="institutes" number="19379">AImotion Bavaria</collection>
    <collection role="persons" number="43324">Belzner, Lenz</collection>
  </doc>
</export-example>
