<?xml version="1.0" encoding="utf-8"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hal="http://hal.archives-ouvertes.fr/" xmlns:gml="http://www.opengis.net/gml/3.3/" xmlns:gmlce="http://www.opengis.net/gml/3.3/ce" version="1.1" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 http://api.archives-ouvertes.fr/documents/aofr-sword.xsd">
  <teiHeader>
    <fileDesc>
      <titleStmt>
        <title>HAL TEI export of hal-03416247</title>
      </titleStmt>
      <publicationStmt>
        <distributor>CCSD</distributor>
        <availability status="restricted">
          <licence target="https://creativecommons.org/publicdomain/zero/1.0/">CC0 1.0 - Universal</licence>
        </availability>
        <date when="2026-05-25T01:27:36+02:00"/>
      </publicationStmt>
      <sourceDesc>
        <p part="N">HAL API Platform</p>
      </sourceDesc>
    </fileDesc>
  </teiHeader>
  <text>
    <body>
      <listBibl>
        <biblFull>
          <titleStmt>
            <title xml:lang="en">Concave Utility Reinforcement Learning: the Mean-field Game viewpoint</title>
            <author role="aut">
              <persName>
                <forename type="first">Matthieu</forename>
                <surname>Geist</surname>
              </persName>
              <idno type="idhal" notation="numeric">790158</idno>
              <idno type="halauthorid" notation="string">7492-790158</idno>
              <idno type="IDREF">https://www.idref.fr/142341819</idno>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Julien</forename>
                <surname>Pérolat</surname>
              </persName>
              <idno type="halauthorid">33064-0</idno>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Mathieu</forename>
                <surname>Laurière</surname>
              </persName>
              <idno type="halauthorid">822232-0</idno>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Romuald</forename>
                <surname>Elie</surname>
              </persName>
              <idno type="halauthorid">123098-0</idno>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Sarah</forename>
                <surname>Perrin</surname>
              </persName>
              <idno type="halauthorid">55803-0</idno>
              <affiliation ref="#struct-1042631"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Olivier</forename>
                <surname>Bachem</surname>
              </persName>
              <idno type="halauthorid">2166351-0</idno>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Rémi</forename>
                <surname>Munos</surname>
              </persName>
              <idno type="halauthorid">181249-0</idno>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Olivier</forename>
                <surname>Pietquin</surname>
              </persName>
              <idno type="halauthorid">23089-0</idno>
            </author>
            <editor role="depositor">
              <persName>
                <forename>Sarah</forename>
                <surname>Perrin</surname>
              </persName>
              <email type="md5">b997a633abf6f88ff14b7658d01216f4</email>
              <email type="domain">inria.fr</email>
            </editor>
          </titleStmt>
          <editionStmt>
            <edition n="v1" type="current">
              <date type="whenSubmitted">2021-11-05 10:44:13</date>
              <date type="whenModified">2026-05-06 09:34:02</date>
              <date type="whenReleased">2021-11-05 10:44:13</date>
              <date type="whenProduced">2021-11-05</date>
              <ref type="externalLink" target="http://arxiv.org/pdf/2106.03787"/>
            </edition>
            <respStmt>
              <resp>contributor</resp>
              <name key="924376">
                <persName>
                  <forename>Sarah</forename>
                  <surname>Perrin</surname>
                </persName>
                <email type="md5">b997a633abf6f88ff14b7658d01216f4</email>
                <email type="domain">inria.fr</email>
              </name>
            </respStmt>
          </editionStmt>
          <publicationStmt>
            <distributor>CCSD</distributor>
            <idno type="halId">hal-03416247</idno>
            <idno type="halUri">https://hal.science/hal-03416247</idno>
            <idno type="halBibtex">geist:hal-03416247</idno>
            <idno type="halRefHtml">2021</idno>
            <idno type="halRef">2021</idno>
            <availability status="restricted"/>
          </publicationStmt>
          <seriesStmt>
            <idno type="stamp" n="CNRS">CNRS - Centre national de la recherche scientifique</idno>
            <idno type="stamp" n="INRIA">INRIA - Institut National de Recherche en Informatique et en Automatique</idno>
            <idno type="stamp" n="INRIA-LILLE">INRIA Lille - Nord Europe</idno>
            <idno type="stamp" n="INRIA_TEST">INRIA - Institut National de Recherche en Informatique et en Automatique</idno>
            <idno type="stamp" n="TESTALAIN1">TESTALAIN1</idno>
            <idno type="stamp" n="CRISTAL">Centre de Recherche en Informatique, Signal et Automatique de Lille (CRISTAL)</idno>
            <idno type="stamp" n="INRIA2">INRIA 2</idno>
            <idno type="stamp" n="UNIV-LILLE">Université de Lille</idno>
            <idno type="stamp" n="TEST-HALCNRS">Collection test HAL CNRS</idno>
            <idno type="stamp" n="CRISTAL-SCOOL" corresp="CRISTAL">SCOOL</idno>
          </seriesStmt>
          <notesStmt/>
          <sourceDesc>
            <biblStruct>
              <analytic>
                <title xml:lang="en">Concave Utility Reinforcement Learning: the Mean-field Game viewpoint</title>
                <author role="aut">
                  <persName>
                    <forename type="first">Matthieu</forename>
                    <surname>Geist</surname>
                  </persName>
                  <idno type="idhal" notation="numeric">790158</idno>
                  <idno type="halauthorid" notation="string">7492-790158</idno>
                  <idno type="IDREF">https://www.idref.fr/142341819</idno>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Julien</forename>
                    <surname>Pérolat</surname>
                  </persName>
                  <idno type="halauthorid">33064-0</idno>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Mathieu</forename>
                    <surname>Laurière</surname>
                  </persName>
                  <idno type="halauthorid">822232-0</idno>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Romuald</forename>
                    <surname>Elie</surname>
                  </persName>
                  <idno type="halauthorid">123098-0</idno>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Sarah</forename>
                    <surname>Perrin</surname>
                  </persName>
                  <idno type="halauthorid">55803-0</idno>
                  <affiliation ref="#struct-1042631"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Olivier</forename>
                    <surname>Bachem</surname>
                  </persName>
                  <idno type="halauthorid">2166351-0</idno>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Rémi</forename>
                    <surname>Munos</surname>
                  </persName>
                  <idno type="halauthorid">181249-0</idno>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Olivier</forename>
                    <surname>Pietquin</surname>
                  </persName>
                  <idno type="halauthorid">23089-0</idno>
                </author>
              </analytic>
              <monogr>
                <imprint/>
              </monogr>
              <idno type="arxiv">2106.03787</idno>
            </biblStruct>
          </sourceDesc>
          <profileDesc>
            <langUsage>
              <language ident="en">English</language>
            </langUsage>
            <textClass>
              <classCode scheme="halDomain" n="info.info-ai">Computer Science [cs]/Artificial Intelligence [cs.AI]</classCode>
              <classCode scheme="halDomain" n="info.info-gt">Computer Science [cs]/Computer Science and Game Theory [cs.GT]</classCode>
              <classCode scheme="halDomain" n="info.info-ma">Computer Science [cs]/Multiagent Systems [cs.MA]</classCode>
              <classCode scheme="halDomain" n="info.info-ne">Computer Science [cs]/Neural and Evolutionary Computing [cs.NE]</classCode>
              <classCode scheme="halTypology" n="UNDEFINED">Preprints, Working Papers, ...</classCode>
              <classCode scheme="halOldTypology" n="UNDEFINED">Preprints, Working Papers, ...</classCode>
              <classCode scheme="halTreeTypology" n="UNDEFINED">Preprints, Working Papers, ...</classCode>
            </textClass>
            <abstract xml:lang="en">
              <p>Concave Utility Reinforcement Learning (CURL) extends RL from linear to concave utilities in the occupancy measure induced by the agent's policy. This encompasses not only RL but also imitation learning and exploration, among others. Yet, this more general paradigm invalidates the classical Bellman equations, and calls for new algorithms. Mean-field Games (MFGs) are a continuous approximation of many-agent RL. They consider the limit case of a continuous distribution of identical agents, anonymous with symmetric interests, and reduce the problem to the study of a single representative agent in interaction with the full population. Our core contribution consists in showing that CURL is a subclass of MFGs. We think this important to bridge together both communities. It also allows to shed light on aspects of both fields: we show the equivalence between concavity in CURL and monotonicity in the associated MFG, between optimality conditions in CURL and Nash equilibrium in MFG, or that Fictitious Play (FP) for this class of MFGs is simply Frank-Wolfe, bringing the first convergence rate for discrete-time FP for MFGs. We also experimentally demonstrate that, using algorithms recently introduced for solving MFGs, we can address the CURL problem more efficiently.</p>
            </abstract>
          </profileDesc>
        </biblFull>
      </listBibl>
    </body>
    <back>
      <listOrg type="structures">
        <org type="researchteam" xml:id="struct-1042631" status="VALID">
          <idno type="RNSR">202023603Y</idno>
          <idno type="ROR">https://ror.org/025wa0q37</idno>
          <orgName>Prise de décision séquentielle dans l’incertain</orgName>
          <orgName type="acronym">Scool</orgName>
          <date type="start">2020-07-01</date>
          <date type="end">2027-12-31</date>
          <desc>
            <address>
              <addrLine>Sequential decision making under uncertainty problemParc Scientifique de la Haute Borne 40, avenue Halley Bât.A, Park Plaza 59650 Villeneuve d'Ascq</addrLine>
              <country key="FR"/>
            </address>
          </desc>
          <listRelation>
            <relation active="#struct-104752" type="direct"/>
            <relation active="#struct-300009" type="indirect"/>
            <relation active="#struct-410272" type="direct"/>
            <relation name="UMR9189" active="#struct-120930" type="indirect"/>
            <relation name="UMR9189" active="#struct-374570" type="indirect"/>
            <relation name="UMR9189" active="#struct-441569" type="indirect"/>
          </listRelation>
        </org>
        <org type="laboratory" xml:id="struct-104752" status="VALID">
          <idno type="RNSR">200818245B</idno>
          <idno type="ROR">https://ror.org/04eej9726</idno>
          <orgName>Centre Inria de l'Université de Lille</orgName>
          <desc>
            <address>
              <addrLine>Parc Scientifique de la Haute Borne 40, avenue Halley Bât.A, Park Plaza 59650 Villeneuve d'Ascq</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/lille/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-300009" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-300009" status="VALID">
          <idno type="ROR">https://ror.org/02kvxyf05</idno>
          <orgName>Institut National de Recherche en Informatique et en Automatique</orgName>
          <orgName type="acronym">Inria</orgName>
          <desc>
            <address>
              <addrLine>Domaine de VoluceauRocquencourt - BP 10578153 Le Chesnay Cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/en/</ref>
          </desc>
        </org>
        <org type="laboratory" xml:id="struct-410272" status="VALID">
          <idno type="IdRef">18388695X</idno>
          <idno type="RNSR">201521249L</idno>
          <idno type="ROR">https://ror.org/05vrs3189</idno>
          <idno type="Wikidata">Q116959497</idno>
          <orgName>Centre de Recherche en Informatique, Signal et Automatique de Lille - UMR 9189</orgName>
          <orgName type="acronym">CRIStAL</orgName>
          <date type="start">2015-01-01</date>
          <desc>
            <address>
              <addrLine>Université de Lille - Campus scientifique - Bâtiment ESPRIT - Avenue Henri Poincaré - 59655 Villeneuve d’Ascq</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.cristal.univ-lille.fr/</ref>
          </desc>
          <listRelation>
            <relation name="UMR9189" active="#struct-120930" type="direct"/>
            <relation name="UMR9189" active="#struct-374570" type="direct"/>
            <relation name="UMR9189" active="#struct-441569" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-120930" status="VALID">
          <idno type="IdRef">256304629</idno>
          <idno type="ISNI">0000000122034461</idno>
          <idno type="ROR">https://ror.org/01x441g73</idno>
          <orgName>Centrale Lille</orgName>
          <desc>
            <address>
              <addrLine>École Centrale de Lille - Cité Scientifique - CS 20048 59651 Villeneuve d'Ascq Cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://centralelille.fr/</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-374570" status="VALID">
          <idno type="IdRef">223446556</idno>
          <idno type="ISNI">0000 0001 2242 6780</idno>
          <idno type="ROR">https://ror.org/02kzqn938</idno>
          <idno type="Wikidata">Q3551621</idno>
          <orgName>Université de Lille</orgName>
          <desc>
            <address>
              <addrLine>EPE Université de Lille. -- 42 rue Paul Duez, 59000 Lille</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.univ-lille.fr/</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-441569" status="VALID">
          <idno type="IdRef">02636817X</idno>
          <idno type="ISNI">0000000122597504</idno>
          <idno type="ROR">https://ror.org/02feahw73</idno>
          <orgName>Centre National de la Recherche Scientifique</orgName>
          <orgName type="acronym">CNRS</orgName>
          <date type="start">1939-10-19</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.cnrs.fr/</ref>
          </desc>
        </org>
      </listOrg>
    </back>
  </text>
</TEI>