<?xml version="1.0" encoding="utf-8"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hal="http://hal.archives-ouvertes.fr/" xmlns:gml="http://www.opengis.net/gml/3.3/" xmlns:gmlce="http://www.opengis.net/gml/3.3/ce" version="1.1" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 http://api.archives-ouvertes.fr/documents/aofr-sword.xsd">
  <teiHeader>
    <fileDesc>
      <titleStmt>
        <title>HAL TEI export of hal-04844089</title>
      </titleStmt>
      <publicationStmt>
        <distributor>CCSD</distributor>
        <availability status="restricted">
          <licence target="https://creativecommons.org/publicdomain/zero/1.0/">CC0 1.0 - Universal</licence>
        </availability>
        <date when="2026-05-17T18:31:55+02:00"/>
      </publicationStmt>
      <sourceDesc>
        <p part="N">HAL API Platform</p>
      </sourceDesc>
    </fileDesc>
  </teiHeader>
  <text>
    <body>
      <listBibl>
        <biblFull>
          <titleStmt>
            <title xml:lang="en">SAC-GLAM: Improving Online RL for LLM agents with Soft Actor-Critic and Hindsight Relabeling</title>
            <author role="aut">
              <persName>
                <forename type="first">Loris</forename>
                <surname>Gaven</surname>
              </persName>
              <idno type="halauthorid">3344841-0</idno>
              <affiliation ref="#struct-60156"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Clément</forename>
                <surname>Romac</surname>
              </persName>
              <email type="md5">b2ebc422b6f7b502aea749eace716fce</email>
              <email type="domain">ynov.com</email>
              <idno type="idhal" notation="string">clement-romac</idno>
              <idno type="idhal" notation="numeric">750773</idno>
              <idno type="halauthorid" notation="string">53243-750773</idno>
              <idno type="GOOGLE SCHOLAR">https://scholar.google.fr/citations?user=L7YK6t0AAAAJ&amp;hl=fr</idno>
              <affiliation ref="#struct-1066038"/>
              <affiliation ref="#struct-60156"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Thomas</forename>
                <surname>Carta</surname>
              </persName>
              <email type="md5">ee2fcc2ebc1f69d1d251439950ba6768</email>
              <email type="domain">polytechnique.edu</email>
              <idno type="idhal" notation="string">thomas-carta</idno>
              <idno type="idhal" notation="numeric">750836</idno>
              <idno type="halauthorid" notation="string">55287-750836</idno>
              <idno type="ORCID">https://orcid.org/0000-0003-3145-549X</idno>
              <affiliation ref="#struct-60156"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Sylvain</forename>
                <surname>Lamprier</surname>
              </persName>
              <idno type="halauthorid">30958-0</idno>
              <affiliation ref="#struct-74911"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Olivier</forename>
                <surname>Sigaud</surname>
              </persName>
              <email type="md5">1a3a7a628849e00426cd103ac2d585cb</email>
              <email type="domain">upmc.fr</email>
              <idno type="idhal" notation="string">olivier-sigaud</idno>
              <idno type="idhal" notation="numeric">14932</idno>
              <idno type="halauthorid" notation="string">15986-14932</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-8544-0229</idno>
              <idno type="IDREF">https://www.idref.fr/072724714</idno>
              <idno type="ARXIV">https://arxiv.org/a/sigaud_o_1</idno>
              <idno type="ISNI">http://isni.org/isni/0000000073281983</idno>
              <idno type="VIAF">https://viaf.org/viaf/43669783</idno>
              <idno type="RESEARCHERID">http://www.researcherid.com/rid/FWG-2898-2022</idno>
              <affiliation ref="#struct-541937"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Pierre-Yves</forename>
                <surname>Oudeyer</surname>
              </persName>
              <email type="md5">9ff76693e026f7a72312fc2b99729d8a</email>
              <email type="domain">inria.fr</email>
              <idno type="idhal" notation="string">pyoudeyer</idno>
              <idno type="idhal" notation="numeric">6675</idno>
              <idno type="halauthorid" notation="string">2543221-6675</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-9404-7613</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-1277-130X</idno>
              <idno type="IDREF">https://www.idref.fr/081674481</idno>
              <affiliation ref="#struct-60156"/>
            </author>
            <editor role="depositor">
              <persName>
                <forename>Clément</forename>
                <surname>ROMAC</surname>
              </persName>
              <email type="md5">d95bc3b9eb4fd3a4520aab0837c2d0ef</email>
              <email type="domain">gmail.com</email>
            </editor>
          </titleStmt>
          <editionStmt>
            <edition n="v1" type="current">
              <date type="whenSubmitted">2024-12-17 17:42:04</date>
              <date type="whenModified">2025-06-12 03:27:02</date>
              <date type="whenReleased">2024-12-17 17:42:04</date>
              <date type="whenProduced">2024-12-15</date>
              <ref type="externalLink" target="http://arxiv.org/pdf/2410.12481"/>
            </edition>
            <respStmt>
              <resp>contributor</resp>
              <name key="710794">
                <persName>
                  <forename>Clément</forename>
                  <surname>ROMAC</surname>
                </persName>
                <email type="md5">d95bc3b9eb4fd3a4520aab0837c2d0ef</email>
                <email type="domain">gmail.com</email>
              </name>
            </respStmt>
          </editionStmt>
          <publicationStmt>
            <distributor>CCSD</distributor>
            <idno type="halId">hal-04844089</idno>
            <idno type="halUri">https://hal.science/hal-04844089</idno>
            <idno type="halBibtex">gaven:hal-04844089</idno>
            <idno type="halRefHtml">&lt;i&gt;IMOL 2024 - Intrinsically Motivated Open-ended Learning (Workshop at Neurips)&lt;/i&gt;, Dec 2024, Vancouver, Canada. 2024</idno>
            <idno type="halRef">IMOL 2024 - Intrinsically Motivated Open-ended Learning (Workshop at Neurips), Dec 2024, Vancouver, Canada. 2024</idno>
            <availability status="restricted"/>
          </publicationStmt>
          <seriesStmt>
            <idno type="stamp" n="CNRS">CNRS - Centre national de la recherche scientifique</idno>
            <idno type="stamp" n="INRIA">INRIA - Institut National de Recherche en Informatique et en Automatique</idno>
            <idno type="stamp" n="UNIV-ANGERS">Université d'Angers</idno>
            <idno type="stamp" n="INRIA-BORDEAUX">INRIA Bordeaux - Sud-Ouest</idno>
            <idno type="stamp" n="INRIA_TEST">INRIA - Institut National de Recherche en Informatique et en Automatique</idno>
            <idno type="stamp" n="ISIR" corresp="SORBONNE-UNIVERSITE">Institut des Systèmes Intelligents et de Robotique</idno>
            <idno type="stamp" n="TESTALAIN1">TESTALAIN1</idno>
            <idno type="stamp" n="ENSTA_U2IS">Unité d'Informatique et d'Ingénierie des Systèmes (U2IS)</idno>
            <idno type="stamp" n="INRIA2">INRIA 2</idno>
            <idno type="stamp" n="SORBONNE-UNIVERSITE">Sorbonne Université</idno>
            <idno type="stamp" n="SORBONNE-UNIV" corresp="SORBONNE-UNIVERSITE">Sorbonne Université 01/01/2018</idno>
            <idno type="stamp" n="SU-SCIENCES" corresp="SORBONNE-UNIVERSITE">Faculté des Sciences de Sorbonne Université</idno>
            <idno type="stamp" n="IP_PARIS">Institut Polytechnique de Paris</idno>
            <idno type="stamp" n="SU-TI">Sorbonne Université - Texte Intégral</idno>
            <idno type="stamp" n="ALLIANCE-SU"> Alliance Sorbonne Université</idno>
            <idno type="stamp" n="ISIR_MLIA" corresp="ISIR">Machine Learning and Information Access</idno>
            <idno type="stamp" n="INRIA-ETATSUNIS">Copublications Inria-Etats-Unis</idno>
            <idno type="stamp" n="SUPRA_INGENIERIE">Ingénierie</idno>
            <idno type="stamp" n="ENSTA-PARIS">ENSTA-PARIS</idno>
            <idno type="stamp" n="ENSTA">Ensta</idno>
          </seriesStmt>
          <notesStmt>
            <note type="audience" n="2">International</note>
            <note type="invited" n="0">No</note>
            <note type="popular" n="0">No</note>
            <note type="peer" n="1">Yes</note>
            <note type="proceedings" n="0">No</note>
          </notesStmt>
          <sourceDesc>
            <biblStruct>
              <analytic>
                <title xml:lang="en">SAC-GLAM: Improving Online RL for LLM agents with Soft Actor-Critic and Hindsight Relabeling</title>
                <author role="aut">
                  <persName>
                    <forename type="first">Loris</forename>
                    <surname>Gaven</surname>
                  </persName>
                  <idno type="halauthorid">3344841-0</idno>
                  <affiliation ref="#struct-60156"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Clément</forename>
                    <surname>Romac</surname>
                  </persName>
                  <email type="md5">b2ebc422b6f7b502aea749eace716fce</email>
                  <email type="domain">ynov.com</email>
                  <idno type="idhal" notation="string">clement-romac</idno>
                  <idno type="idhal" notation="numeric">750773</idno>
                  <idno type="halauthorid" notation="string">53243-750773</idno>
                  <idno type="GOOGLE SCHOLAR">https://scholar.google.fr/citations?user=L7YK6t0AAAAJ&amp;hl=fr</idno>
                  <affiliation ref="#struct-1066038"/>
                  <affiliation ref="#struct-60156"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Thomas</forename>
                    <surname>Carta</surname>
                  </persName>
                  <email type="md5">ee2fcc2ebc1f69d1d251439950ba6768</email>
                  <email type="domain">polytechnique.edu</email>
                  <idno type="idhal" notation="string">thomas-carta</idno>
                  <idno type="idhal" notation="numeric">750836</idno>
                  <idno type="halauthorid" notation="string">55287-750836</idno>
                  <idno type="ORCID">https://orcid.org/0000-0003-3145-549X</idno>
                  <affiliation ref="#struct-60156"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Sylvain</forename>
                    <surname>Lamprier</surname>
                  </persName>
                  <idno type="halauthorid">30958-0</idno>
                  <affiliation ref="#struct-74911"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Olivier</forename>
                    <surname>Sigaud</surname>
                  </persName>
                  <email type="md5">1a3a7a628849e00426cd103ac2d585cb</email>
                  <email type="domain">upmc.fr</email>
                  <idno type="idhal" notation="string">olivier-sigaud</idno>
                  <idno type="idhal" notation="numeric">14932</idno>
                  <idno type="halauthorid" notation="string">15986-14932</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-8544-0229</idno>
                  <idno type="IDREF">https://www.idref.fr/072724714</idno>
                  <idno type="ARXIV">https://arxiv.org/a/sigaud_o_1</idno>
                  <idno type="ISNI">http://isni.org/isni/0000000073281983</idno>
                  <idno type="VIAF">https://viaf.org/viaf/43669783</idno>
                  <idno type="RESEARCHERID">http://www.researcherid.com/rid/FWG-2898-2022</idno>
                  <affiliation ref="#struct-541937"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Pierre-Yves</forename>
                    <surname>Oudeyer</surname>
                  </persName>
                  <email type="md5">9ff76693e026f7a72312fc2b99729d8a</email>
                  <email type="domain">inria.fr</email>
                  <idno type="idhal" notation="string">pyoudeyer</idno>
                  <idno type="idhal" notation="numeric">6675</idno>
                  <idno type="halauthorid" notation="string">2543221-6675</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-9404-7613</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-1277-130X</idno>
                  <idno type="IDREF">https://www.idref.fr/081674481</idno>
                  <affiliation ref="#struct-60156"/>
                </author>
              </analytic>
              <monogr>
                <meeting>
                  <title>IMOL 2024 - Intrinsically Motivated Open-ended Learning (Workshop at Neurips)</title>
                  <date type="start">2024-12-15</date>
                  <date type="end">2024-12-15</date>
                  <settlement>Vancouver</settlement>
                  <country key="CA">Canada</country>
                </meeting>
                <imprint>
                  <date type="datePub">2024-10-16</date>
                </imprint>
              </monogr>
              <idno type="arxiv">2410.12481</idno>
            </biblStruct>
          </sourceDesc>
          <profileDesc>
            <langUsage>
              <language ident="en">English</language>
            </langUsage>
            <textClass>
              <classCode scheme="halDomain" n="info.info-lg">Computer Science [cs]/Machine Learning [cs.LG]</classCode>
              <classCode scheme="halDomain" n="info.info-ai">Computer Science [cs]/Artificial Intelligence [cs.AI]</classCode>
              <classCode scheme="halTypology" n="POSTER">Poster communications</classCode>
              <classCode scheme="halOldTypology" n="POSTER">Poster communications</classCode>
              <classCode scheme="halTreeTypology" n="POSTER">Poster communications</classCode>
            </textClass>
            <abstract xml:lang="en">
              <p>The past years have seen Large Language Models (LLMs) strive not only as generative models but also as agents solving textual sequential decision-making tasks. When facing complex environments where their zero-shot abilities are insufficient, recent work showed online Reinforcement Learning (RL) could be used for the LLM agent to discover and learn efficient strategies interactively. However, most prior work sticks to on-policy algorithms, which greatly reduces the scope of methods such agents could use for both exploration and exploitation, such as experience replay and hindsight relabeling. Yet, such methods may be key for LLM learning agents, and in particular when designing autonomous intrinsically motivated agents sampling and pursuing their own goals (i.e. autotelic agents). This paper presents and studies an adaptation of Soft Actor-Critic and hindsight relabeling to LLM agents. Our method not only paves the path towards autotelic LLM agents that learn online but can also outperform on-policy methods in more classic multi-goal RL environments.</p>
            </abstract>
          </profileDesc>
        </biblFull>
      </listBibl>
    </body>
    <back>
      <listOrg type="structures">
        <org type="researchteam" xml:id="struct-60156" status="OLD">
          <idno type="RNSR">200820949R</idno>
          <orgName>Flowing Epigenetic Robots and Systems</orgName>
          <orgName type="acronym">Flowers</orgName>
          <date type="start">2011-01-01</date>
          <date type="end">2025-02-28</date>
          <desc>
            <address>
              <addrLine>200, avenue de la Vieille Tour 33405 Talence cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://flowers.inria.fr</ref>
          </desc>
          <listRelation>
            <relation active="#struct-104751" type="direct"/>
            <relation active="#struct-300009" type="indirect"/>
            <relation active="#struct-234188" type="direct"/>
            <relation active="#struct-300065" type="indirect"/>
            <relation active="#struct-563936" type="indirect"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-1066038" status="VALID">
          <orgName>Hugging Face</orgName>
          <desc>
            <address>
              <country key="US"/>
            </address>
            <ref type="url">https://huggingface.co/</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-74911" status="VALID">
          <idno type="IdRef">026402920</idno>
          <idno type="ISNI">0000000122483363</idno>
          <idno type="ROR">https://ror.org/04yrqp957</idno>
          <orgName>Université d'Angers</orgName>
          <orgName type="acronym">UA</orgName>
          <date type="start">1971-10-25</date>
          <desc>
            <address>
              <addrLine>Université d'Angers - 40 Rue de Rennes, BP 73532 - 49035 Angers CEDEX 01</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.univ-angers.fr/</ref>
          </desc>
        </org>
        <org type="laboratory" xml:id="struct-541937" status="VALID">
          <idno type="IdRef">241179122</idno>
          <idno type="RNSR">200918463J</idno>
          <idno type="ROR">05neq8668</idno>
          <orgName>Institut des Systèmes Intelligents et de Robotique</orgName>
          <orgName type="acronym">ISIR</orgName>
          <date type="start">2018-01-01</date>
          <desc>
            <address>
              <addrLine>Sorbonne-Université, Boite courrier 173 4 Place JUSSIEU 75252 Paris cedex 05</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.isir.upmc.fr</ref>
          </desc>
          <listRelation>
            <relation active="#struct-413221" type="direct"/>
            <relation name="UMR7222" active="#struct-441569" type="direct"/>
          </listRelation>
        </org>
        <org type="laboratory" xml:id="struct-104751" status="VALID">
          <idno type="RNSR">200818243Z</idno>
          <idno type="ROR">https://ror.org/03tjcj052</idno>
          <orgName>Centre Inria de l'Université de Bordeaux</orgName>
          <desc>
            <address>
              <addrLine>200, avenue de la Vieille Tour, 33405 Talence</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/bordeaux/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-300009" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-300009" status="VALID">
          <idno type="ROR">https://ror.org/02kvxyf05</idno>
          <orgName>Institut National de Recherche en Informatique et en Automatique</orgName>
          <orgName type="acronym">Inria</orgName>
          <desc>
            <address>
              <addrLine>Domaine de VoluceauRocquencourt - BP 10578153 Le Chesnay Cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/en/</ref>
          </desc>
        </org>
        <org type="regrouplaboratory" xml:id="struct-234188" status="OLD">
          <idno type="IdRef">203564529</idno>
          <idno type="RNSR">200920620D</idno>
          <orgName>Unité d'Informatique et d'Ingénierie des Systèmes</orgName>
          <orgName type="acronym">U2IS</orgName>
          <date type="end">2024-12-31</date>
          <desc>
            <address>
              <addrLine>828, boulevard des Maréchaux, 91762 Palaiseau CEDEX</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://u2is.ensta-paris.fr/index.php?lang=fr</ref>
          </desc>
          <listRelation>
            <relation active="#struct-300065" type="direct"/>
            <relation active="#struct-563936" type="indirect"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-300065" status="OLD">
          <idno type="IdRef">028139577</idno>
          <idno type="ISNI">0000000122070120</idno>
          <idno type="ROR">https://ror.org/0309cs235</idno>
          <idno type="Wikidata">Q838454</idno>
          <orgName>École Nationale Supérieure de Techniques Avancées</orgName>
          <orgName type="acronym">ENSTA Paris</orgName>
          <date type="end">2024-12-31</date>
          <desc>
            <address>
              <addrLine>828, boulevard des Maréchaux 91120 Palaiseau</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.ensta-paris.fr/fr</ref>
          </desc>
          <listRelation>
            <relation active="#struct-563936" type="direct"/>
          </listRelation>
        </org>
        <org type="regroupinstitution" xml:id="struct-563936" status="VALID">
          <idno type="IdRef">238327159</idno>
          <idno type="ISNI">0000000502717600</idno>
          <idno type="ROR">https://ror.org/042tfbd02</idno>
          <idno type="Wikidata">Q48759778</idno>
          <orgName>Institut Polytechnique de Paris</orgName>
          <orgName type="acronym">IP Paris</orgName>
          <date type="start">2019-06-02</date>
          <desc>
            <address>
              <addrLine>Route de Saclay, 91120 Palaiseau Cedex, France</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.ip-paris.fr</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-413221" status="VALID">
          <idno type="IdRef">221333754</idno>
          <idno type="ROR">https://ror.org/02en5vm52</idno>
          <orgName>Sorbonne Université</orgName>
          <orgName type="acronym">SU</orgName>
          <date type="start">2018-01-01</date>
          <desc>
            <address>
              <addrLine>21 rue de l’École de médecine - 75006 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.sorbonne-universite.fr/</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-441569" status="VALID">
          <idno type="IdRef">02636817X</idno>
          <idno type="ISNI">0000000122597504</idno>
          <idno type="ROR">https://ror.org/02feahw73</idno>
          <orgName>Centre National de la Recherche Scientifique</orgName>
          <orgName type="acronym">CNRS</orgName>
          <date type="start">1939-10-19</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.cnrs.fr/</ref>
          </desc>
        </org>
      </listOrg>
    </back>
  </text>
</TEI>