<?xml version="1.0" encoding="utf-8"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hal="http://hal.archives-ouvertes.fr/" xmlns:gml="http://www.opengis.net/gml/3.3/" xmlns:gmlce="http://www.opengis.net/gml/3.3/ce" version="1.1" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 http://api.archives-ouvertes.fr/documents/aofr-sword.xsd">
  <teiHeader>
    <fileDesc>
      <titleStmt>
        <title>HAL TEI export of hal-04239313</title>
      </titleStmt>
      <publicationStmt>
        <distributor>CCSD</distributor>
        <availability status="restricted">
          <licence target="https://creativecommons.org/publicdomain/zero/1.0/">CC0 1.0 - Universal</licence>
        </availability>
        <date when="2026-05-02T10:00:21+02:00"/>
      </publicationStmt>
      <sourceDesc>
        <p part="N">HAL API Platform</p>
      </sourceDesc>
    </fileDesc>
  </teiHeader>
  <text>
    <body>
      <listBibl>
        <biblFull>
          <titleStmt>
            <title xml:lang="en">Discovering and Exploiting Sparse Rewards in a Learned Behavior Space</title>
            <author role="aut">
              <persName>
                <forename type="first">Giuseppe</forename>
                <surname>Paolo</surname>
              </persName>
              <email type="md5">afc17c1a80c629103cd36a9cdf77d788</email>
              <email type="domain">softbankrobotics.com</email>
              <idno type="idhal" notation="string">gpaolo</idno>
              <idno type="idhal" notation="numeric">735825</idno>
              <idno type="halauthorid" notation="string">49437-735825</idno>
              <idno type="ARXIV">https://arxiv.org/a/paolo_g_1.html</idno>
              <idno type="ORCID">https://orcid.org/0000-0003-4201-5967</idno>
              <idno type="GOOGLE SCHOLAR">https://scholar.google.fr/citations?user=khT6tDsAAAAJ&amp;hl=it&amp;authuser=1</idno>
              <idno type="IDREF">https://www.idref.fr/261849867</idno>
              <affiliation ref="#struct-541937"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Miranda</forename>
                <surname>Coninx</surname>
              </persName>
              <email type="md5">7f23bda51ae83f6193f2ee40e5e4a994</email>
              <email type="domain">sorbonne-universite.fr</email>
              <idno type="idhal" notation="string">alex-coninx</idno>
              <idno type="idhal" notation="numeric">184690</idno>
              <idno type="halauthorid" notation="string">2772260-184690</idno>
              <idno type="ARXIV">https://arxiv.org/a/coninx_a_1</idno>
              <idno type="ORCID">https://orcid.org/0000-0001-7992-8183</idno>
              <idno type="IDREF">https://www.idref.fr/166602183</idno>
              <idno type="VIAF">https://viaf.org/viaf/295930989</idno>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Alban</forename>
                <surname>Laflaquière</surname>
              </persName>
              <email type="md5">64851ce9789d8bd8b15b1b4687a19a42</email>
              <email type="domain">softbankrobotics.com</email>
              <idno type="idhal" notation="numeric">1096132</idno>
              <idno type="halauthorid" notation="string">609644-1096132</idno>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Stéphane</forename>
                <surname>Doncieux</surname>
              </persName>
              <email type="md5">b937d1286db3da7e0877aa203468bd7d</email>
              <email type="domain">sorbonne-universite.fr</email>
              <idno type="idhal" notation="string">stephane-doncieux</idno>
              <idno type="idhal" notation="numeric">3909</idno>
              <idno type="halauthorid" notation="string">413189-3909</idno>
              <idno type="ORCID">https://orcid.org/0000-0003-1541-054X</idno>
              <idno type="IDREF">https://www.idref.fr/089428617</idno>
              <idno type="ISNI">http://isni.org/isni/000000010267981X</idno>
              <idno type="VIAF">https://viaf.org/viaf/153201463</idno>
              <affiliation ref="#struct-541937"/>
            </author>
            <editor role="depositor">
              <persName>
                <forename>Giuseppe</forename>
                <surname>Paolo</surname>
              </persName>
              <email type="md5">57f65902d2ac3c9a4585297c441aa339</email>
              <email type="domain">gmail.com</email>
            </editor>
            <funder ref="#projeurop-727318"/>
          </titleStmt>
          <editionStmt>
            <edition n="v1" type="current">
              <date type="whenSubmitted">2023-10-12 15:47:09</date>
              <date type="whenModified">2024-12-09 14:14:51</date>
              <date type="whenReleased">2023-10-13 07:55:18</date>
              <date type="whenProduced">2023</date>
              <date type="whenEndEmbargoed">2023-10-12</date>
              <ref type="file" target="https://hal.science/hal-04239313v1/document">
                <date notBefore="2023-10-12"/>
              </ref>
              <ref type="file" subtype="author" n="1" target="https://hal.science/hal-04239313v1/file/2111.01919.pdf" id="file-4239313-3699126">
                <date notBefore="2023-10-12"/>
              </ref>
              <ref type="externalLink" target="http://arxiv.org/pdf/2111.01919"/>
            </edition>
            <respStmt>
              <resp>contributor</resp>
              <name key="860076">
                <persName>
                  <forename>Giuseppe</forename>
                  <surname>Paolo</surname>
                </persName>
                <email type="md5">57f65902d2ac3c9a4585297c441aa339</email>
                <email type="domain">gmail.com</email>
              </name>
            </respStmt>
          </editionStmt>
          <publicationStmt>
            <distributor>CCSD</distributor>
            <idno type="halId">hal-04239313</idno>
            <idno type="halUri">https://hal.science/hal-04239313</idno>
            <idno type="halBibtex">paolo:hal-04239313</idno>
            <idno type="halRefHtml">&lt;i&gt;Evolutionary Computation&lt;/i&gt;, In press, pp.1-28</idno>
            <idno type="halRef">Evolutionary Computation, In press, pp.1-28</idno>
            <availability status="restricted">
              <licence target="https://about.hal.science/hal-authorisation-v1/">HAL Authorization<ref corresp="#file-4239313-3699126"/></licence>
            </availability>
          </publicationStmt>
          <seriesStmt>
            <idno type="stamp" n="CNRS">CNRS - Centre national de la recherche scientifique</idno>
            <idno type="stamp" n="OPENAIRE">OpenAIRE</idno>
            <idno type="stamp" n="ISIR" corresp="SORBONNE-UNIVERSITE">Institut des Systèmes Intelligents et de Robotique</idno>
            <idno type="stamp" n="SORBONNE-UNIVERSITE">Sorbonne Université</idno>
            <idno type="stamp" n="SORBONNE-UNIV" corresp="SORBONNE-UNIVERSITE">Sorbonne Université 01/01/2018</idno>
            <idno type="stamp" n="SU-SCIENCES" corresp="SORBONNE-UNIVERSITE">Faculté des Sciences de Sorbonne Université</idno>
            <idno type="stamp" n="SU-TI">Sorbonne Université - Texte Intégral</idno>
            <idno type="stamp" n="ISIR_AMAC" corresp="ISIR">Architectures et Modèles de l'Adaptation et de la Cognition</idno>
            <idno type="stamp" n="ALLIANCE-SU"> Alliance Sorbonne Université</idno>
            <idno type="stamp" n="SUPRA_INGENIERIE">Ingénierie</idno>
          </seriesStmt>
          <notesStmt>
            <note type="audience" n="2">International</note>
            <note type="popular" n="0">No</note>
            <note type="peer" n="1">Yes</note>
          </notesStmt>
          <sourceDesc>
            <biblStruct>
              <analytic>
                <title xml:lang="en">Discovering and Exploiting Sparse Rewards in a Learned Behavior Space</title>
                <author role="aut">
                  <persName>
                    <forename type="first">Giuseppe</forename>
                    <surname>Paolo</surname>
                  </persName>
                  <email type="md5">afc17c1a80c629103cd36a9cdf77d788</email>
                  <email type="domain">softbankrobotics.com</email>
                  <idno type="idhal" notation="string">gpaolo</idno>
                  <idno type="idhal" notation="numeric">735825</idno>
                  <idno type="halauthorid" notation="string">49437-735825</idno>
                  <idno type="ARXIV">https://arxiv.org/a/paolo_g_1.html</idno>
                  <idno type="ORCID">https://orcid.org/0000-0003-4201-5967</idno>
                  <idno type="GOOGLE SCHOLAR">https://scholar.google.fr/citations?user=khT6tDsAAAAJ&amp;hl=it&amp;authuser=1</idno>
                  <idno type="IDREF">https://www.idref.fr/261849867</idno>
                  <affiliation ref="#struct-541937"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Miranda</forename>
                    <surname>Coninx</surname>
                  </persName>
                  <email type="md5">7f23bda51ae83f6193f2ee40e5e4a994</email>
                  <email type="domain">sorbonne-universite.fr</email>
                  <idno type="idhal" notation="string">alex-coninx</idno>
                  <idno type="idhal" notation="numeric">184690</idno>
                  <idno type="halauthorid" notation="string">2772260-184690</idno>
                  <idno type="ARXIV">https://arxiv.org/a/coninx_a_1</idno>
                  <idno type="ORCID">https://orcid.org/0000-0001-7992-8183</idno>
                  <idno type="IDREF">https://www.idref.fr/166602183</idno>
                  <idno type="VIAF">https://viaf.org/viaf/295930989</idno>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Alban</forename>
                    <surname>Laflaquière</surname>
                  </persName>
                  <email type="md5">64851ce9789d8bd8b15b1b4687a19a42</email>
                  <email type="domain">softbankrobotics.com</email>
                  <idno type="idhal" notation="numeric">1096132</idno>
                  <idno type="halauthorid" notation="string">609644-1096132</idno>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Stéphane</forename>
                    <surname>Doncieux</surname>
                  </persName>
                  <email type="md5">b937d1286db3da7e0877aa203468bd7d</email>
                  <email type="domain">sorbonne-universite.fr</email>
                  <idno type="idhal" notation="string">stephane-doncieux</idno>
                  <idno type="idhal" notation="numeric">3909</idno>
                  <idno type="halauthorid" notation="string">413189-3909</idno>
                  <idno type="ORCID">https://orcid.org/0000-0003-1541-054X</idno>
                  <idno type="IDREF">https://www.idref.fr/089428617</idno>
                  <idno type="ISNI">http://isni.org/isni/000000010267981X</idno>
                  <idno type="VIAF">https://viaf.org/viaf/153201463</idno>
                  <affiliation ref="#struct-541937"/>
                </author>
              </analytic>
              <monogr>
                <idno type="halJournalId" status="VALID">13175</idno>
                <idno type="issn">1063-6560</idno>
                <idno type="eissn">1530-9304</idno>
                <title level="j">Evolutionary Computation</title>
                <imprint>
                  <publisher>Massachusetts Institute of Technology Press (MIT Press)</publisher>
                  <biblScope unit="pp">1-28</biblScope>
                  <date type="datePub" subtype="inPress">2023</date>
                  <date type="dateEpub">2023-10-06</date>
                </imprint>
              </monogr>
              <idno type="arxiv">2111.01919</idno>
            </biblStruct>
          </sourceDesc>
          <profileDesc>
            <langUsage>
              <language ident="en">English</language>
            </langUsage>
            <textClass>
              <keywords scheme="author">
                <term xml:lang="en">Sparse rewards</term>
                <term xml:lang="en">Evolutionary algorithm</term>
                <term xml:lang="en">Novelty Search</term>
                <term xml:lang="en">Qualit-Diversity</term>
                <term xml:lang="en">Reinforcement Learning</term>
              </keywords>
              <classCode scheme="halDomain" n="info">Computer Science [cs]</classCode>
              <classCode scheme="halDomain" n="info.info-ai">Computer Science [cs]/Artificial Intelligence [cs.AI]</classCode>
              <classCode scheme="halTypology" n="ART">Journal articles</classCode>
              <classCode scheme="halOldTypology" n="ART">Journal articles</classCode>
              <classCode scheme="halTreeTypology" n="ART">Journal articles</classCode>
            </textClass>
            <abstract xml:lang="en">
              <p>Learning optimal policies in sparse rewards settings is difficult as the learning agent has little to no feedback on the quality of its actions. In these situations, a good strategy is to focus on exploration, hopefully leading to the discovery of a reward signal to improve on. A learning algorithm capable of dealing with this kind of settings has to be able to (1) explore possible agent behaviors and (2) exploit any possible discovered reward. Efficient exploration algorithms have been proposed that require to define a behavior space, that associates to an agent its resulting behavior in a space that is known to be worth exploring. The need to define this space is a limitation of these algorithms. In this work, we introduce STAX, an algorithm designed to learn a behavior space on-the-fly and to explore it while efficiently optimizing any reward discovered. It does so by separating the exploration and learning of the behavior space from the exploitation of the reward through an alternating two-steps process. In the first step, STAX builds a repertoire of diverse policies while learning a low-dimensional representation of the high-dimensional observations generated during the policies evaluation. In the exploitation step, emitters are used to optimize the performance of the discovered rewarding solutions. Experiments conducted on three different sparse reward environments show that STAX performs comparably to existing baselines while requiring much less prior information about the task as it autonomously builds the behavior space.</p>
            </abstract>
          </profileDesc>
        </biblFull>
      </listBibl>
    </body>
    <back>
      <listOrg type="structures">
        <org type="laboratory" xml:id="struct-541937" status="VALID">
          <idno type="IdRef">241179122</idno>
          <idno type="RNSR">200918463J</idno>
          <idno type="ROR">05neq8668</idno>
          <orgName>Institut des Systèmes Intelligents et de Robotique</orgName>
          <orgName type="acronym">ISIR</orgName>
          <date type="start">2018-01-01</date>
          <desc>
            <address>
              <addrLine>Sorbonne-Université, Boite courrier 173 4 Place JUSSIEU 75252 Paris cedex 05</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.isir.upmc.fr</ref>
          </desc>
          <listRelation>
            <relation active="#struct-413221" type="direct"/>
            <relation name="UMR7222" active="#struct-441569" type="direct"/>
          </listRelation>
        </org>
        <org type="regroupinstitution" xml:id="struct-413221" status="VALID">
          <idno type="IdRef">221333754</idno>
          <idno type="ROR">https://ror.org/02en5vm52</idno>
          <orgName>Sorbonne Université</orgName>
          <orgName type="acronym">SU</orgName>
          <date type="start">2018-01-01</date>
          <desc>
            <address>
              <addrLine>21 rue de l’École de médecine - 75006 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.sorbonne-universite.fr/</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-441569" status="VALID">
          <idno type="IdRef">02636817X</idno>
          <idno type="ISNI">0000000122597504</idno>
          <idno type="ROR">https://ror.org/02feahw73</idno>
          <orgName>Centre National de la Recherche Scientifique</orgName>
          <orgName type="acronym">CNRS</orgName>
          <date type="start">1939-10-19</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.cnrs.fr/</ref>
          </desc>
        </org>
      </listOrg>
      <listOrg type="projects">
        <org type="europeanProject" xml:id="projeurop-727318" status="INCOMING">
          <desc>European Commission’s Horizon Europe Framework</desc>
        </org>
      </listOrg>
    </back>
  </text>
</TEI>