<?xml version="1.0" encoding="utf-8"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hal="http://hal.archives-ouvertes.fr/" xmlns:gml="http://www.opengis.net/gml/3.3/" xmlns:gmlce="http://www.opengis.net/gml/3.3/ce" version="1.1" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 http://api.archives-ouvertes.fr/documents/aofr-sword.xsd">
  <teiHeader>
    <fileDesc>
      <titleStmt>
        <title>HAL TEI export of hal-03301611</title>
      </titleStmt>
      <publicationStmt>
        <distributor>CCSD</distributor>
        <availability status="restricted">
          <licence target="https://creativecommons.org/publicdomain/zero/1.0/">CC0 1.0 - Universal</licence>
        </availability>
        <date when="2026-05-15T22:43:49+02:00"/>
      </publicationStmt>
      <sourceDesc>
        <p part="N">HAL API Platform</p>
      </sourceDesc>
    </fileDesc>
  </teiHeader>
  <text>
    <body>
      <listBibl>
        <biblFull>
          <titleStmt>
            <title xml:lang="en">Evaluate on-the-job learning dialogue systems and a case study for natural language understanding</title>
            <title xml:lang="fr">Évaluer les systèmes de dialogue apprenant sur le terrain et un cas d'étude pour la compréhension du langage naturel</title>
            <author role="aut">
              <persName>
                <forename type="first">Mathilde</forename>
                <surname>Veron</surname>
              </persName>
              <email type="md5">083d2b0b80af62606e39b9422ce03bb1</email>
              <email type="domain">limsi.fr</email>
              <idno type="idhal" notation="string">mathilde-veron</idno>
              <idno type="idhal" notation="numeric">177634</idno>
              <idno type="halauthorid" notation="string">46333-177634</idno>
              <idno type="IDREF">https://www.idref.fr/267763468</idno>
              <affiliation ref="#struct-559909"/>
              <affiliation ref="#struct-1041998"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Sophie</forename>
                <surname>Rosset</surname>
              </persName>
              <email type="md5">dbb4013fd09f2b07c31e3a8e3878f7bc</email>
              <email type="domain">limsi.fr</email>
              <idno type="idhal" notation="string">sophie-rosset</idno>
              <idno type="idhal" notation="numeric">14913</idno>
              <idno type="halauthorid" notation="string">11264-14913</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-6865-4989</idno>
              <idno type="IDREF">https://www.idref.fr/137157835</idno>
              <idno type="ISNI">http://isni.org/isni/0000000357034976</idno>
              <idno type="VIAF">https://viaf.org/viaf/193001246</idno>
              <affiliation ref="#struct-1041998"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Olivier</forename>
                <surname>Galibert</surname>
              </persName>
              <email type="md5">9cfed28af822110ce61e72152af2ac15</email>
              <email type="domain">limsi.fr</email>
              <idno type="idhal" notation="numeric">1034095</idno>
              <idno type="halauthorid" notation="string">28330-1034095</idno>
              <idno type="ISNI">http://isni.org/isni/0000000357950720</idno>
              <idno type="VIAF">https://viaf.org/viaf/205003909</idno>
              <idno type="IDREF">https://www.idref.fr/136783457</idno>
              <affiliation ref="#struct-559909"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Guillaume</forename>
                <surname>Bernard</surname>
              </persName>
              <email type="md5">bdb224602bd1322385252e8f82731ac7</email>
              <email type="domain">univ-lr.fr</email>
              <idno type="idhal" notation="string">guillaume-bernard</idno>
              <idno type="idhal" notation="numeric">747506</idno>
              <idno type="halauthorid" notation="string">472262-747506</idno>
              <idno type="ORCID">https://orcid.org/0000-0001-5945-4865</idno>
              <idno type="IDREF">https://www.idref.fr/270262768</idno>
              <affiliation ref="#struct-559909"/>
            </author>
            <editor role="depositor">
              <persName>
                <forename>Mathilde</forename>
                <surname>Veron</surname>
              </persName>
              <email type="md5">adfc20a1df37a6e4248e88f3991b4944</email>
              <email type="domain">linsi.upsaclay.fr.invalid</email>
            </editor>
          </titleStmt>
          <editionStmt>
            <edition n="v1" type="current">
              <date type="whenSubmitted">2021-07-27 15:40:42</date>
              <date type="whenWritten">2020-10-14</date>
              <date type="whenModified">2026-03-27 11:19:52</date>
              <date type="whenReleased">2021-07-28 13:33:05</date>
              <date type="whenProduced">2020-12-11</date>
              <date type="whenEndEmbargoed">2021-07-27</date>
              <ref type="file" target="https://hal.science/hal-03301611v1/document">
                <date notBefore="2021-07-27"/>
              </ref>
              <ref type="file" subtype="author" n="1" target="https://hal.science/hal-03301611v1/file/NeurIPS_2020_HLDS_on_the_job_NLU_v4_camera_ready.pdf" id="file-3301611-2900203">
                <date notBefore="2021-07-27"/>
              </ref>
              <ref type="externalLink" target="http://arxiv.org/pdf/2102.13589"/>
            </edition>
            <respStmt>
              <resp>contributor</resp>
              <name key="791836">
                <persName>
                  <forename>Mathilde</forename>
                  <surname>Veron</surname>
                </persName>
                <email type="md5">adfc20a1df37a6e4248e88f3991b4944</email>
                <email type="domain">linsi.upsaclay.fr.invalid</email>
              </name>
            </respStmt>
          </editionStmt>
          <publicationStmt>
            <distributor>CCSD</distributor>
            <idno type="halId">hal-03301611</idno>
            <idno type="halUri">https://hal.science/hal-03301611</idno>
            <idno type="halBibtex">veron:hal-03301611</idno>
            <idno type="halRefHtml">&lt;i&gt;Workshop NeurIPS 2020 Human in the Loop Dialogue Systems&lt;/i&gt;, Dec 2020, Virtual only, United States</idno>
            <idno type="halRef">Workshop NeurIPS 2020 Human in the Loop Dialogue Systems, Dec 2020, Virtual only, United States</idno>
            <availability status="restricted">
              <licence target="https://about.hal.science/hal-authorisation-v1/">HAL Authorization<ref corresp="#file-3301611-2900203"/></licence>
            </availability>
          </publicationStmt>
          <seriesStmt>
            <idno type="stamp" n="CNRS">CNRS - Centre national de la recherche scientifique</idno>
            <idno type="stamp" n="LIMSI">Laboratoire d'Informatique pour la Mécanique et les Sciences de l'Ingénieur</idno>
            <idno type="stamp" n="UNIV-PARIS-SACLAY">Université Paris-Saclay</idno>
            <idno type="stamp" n="LNE">Laboratoire National de Métrologie et d’Essais</idno>
            <idno type="stamp" n="TEST-HALCNRS">Collection test HAL CNRS</idno>
            <idno type="stamp" n="UNIVERSITE-PARIS-SACLAY" corresp="UNIV-PARIS-SACLAY">Université Paris-Saclay</idno>
            <idno type="stamp" n="LISN">Laboratoire Interdisciplinaire des Sciences du Numérique</idno>
            <idno type="stamp" n="GS-ENGINEERING">Graduate School Sciences de l'Ingénierie et des Systèmes</idno>
            <idno type="stamp" n="GS-COMPUTER-SCIENCE">Graduate School Computer Science</idno>
            <idno type="stamp" n="LISN-ILES" corresp="LISN">Information, Langue Ecrite et Signée </idno>
            <idno type="stamp" n="LISN-STL">Sciences et Technologies des Langues</idno>
          </seriesStmt>
          <notesStmt>
            <note type="audience" n="2">International</note>
            <note type="invited" n="0">No</note>
            <note type="popular" n="0">No</note>
            <note type="peer" n="1">Yes</note>
            <note type="proceedings" n="0">No</note>
          </notesStmt>
          <sourceDesc>
            <biblStruct>
              <analytic>
                <title xml:lang="en">Evaluate on-the-job learning dialogue systems and a case study for natural language understanding</title>
                <title xml:lang="fr">Évaluer les systèmes de dialogue apprenant sur le terrain et un cas d'étude pour la compréhension du langage naturel</title>
                <author role="aut">
                  <persName>
                    <forename type="first">Mathilde</forename>
                    <surname>Veron</surname>
                  </persName>
                  <email type="md5">083d2b0b80af62606e39b9422ce03bb1</email>
                  <email type="domain">limsi.fr</email>
                  <idno type="idhal" notation="string">mathilde-veron</idno>
                  <idno type="idhal" notation="numeric">177634</idno>
                  <idno type="halauthorid" notation="string">46333-177634</idno>
                  <idno type="IDREF">https://www.idref.fr/267763468</idno>
                  <affiliation ref="#struct-559909"/>
                  <affiliation ref="#struct-1041998"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Sophie</forename>
                    <surname>Rosset</surname>
                  </persName>
                  <email type="md5">dbb4013fd09f2b07c31e3a8e3878f7bc</email>
                  <email type="domain">limsi.fr</email>
                  <idno type="idhal" notation="string">sophie-rosset</idno>
                  <idno type="idhal" notation="numeric">14913</idno>
                  <idno type="halauthorid" notation="string">11264-14913</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-6865-4989</idno>
                  <idno type="IDREF">https://www.idref.fr/137157835</idno>
                  <idno type="ISNI">http://isni.org/isni/0000000357034976</idno>
                  <idno type="VIAF">https://viaf.org/viaf/193001246</idno>
                  <affiliation ref="#struct-1041998"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Olivier</forename>
                    <surname>Galibert</surname>
                  </persName>
                  <email type="md5">9cfed28af822110ce61e72152af2ac15</email>
                  <email type="domain">limsi.fr</email>
                  <idno type="idhal" notation="numeric">1034095</idno>
                  <idno type="halauthorid" notation="string">28330-1034095</idno>
                  <idno type="ISNI">http://isni.org/isni/0000000357950720</idno>
                  <idno type="VIAF">https://viaf.org/viaf/205003909</idno>
                  <idno type="IDREF">https://www.idref.fr/136783457</idno>
                  <affiliation ref="#struct-559909"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Guillaume</forename>
                    <surname>Bernard</surname>
                  </persName>
                  <email type="md5">bdb224602bd1322385252e8f82731ac7</email>
                  <email type="domain">univ-lr.fr</email>
                  <idno type="idhal" notation="string">guillaume-bernard</idno>
                  <idno type="idhal" notation="numeric">747506</idno>
                  <idno type="halauthorid" notation="string">472262-747506</idno>
                  <idno type="ORCID">https://orcid.org/0000-0001-5945-4865</idno>
                  <idno type="IDREF">https://www.idref.fr/270262768</idno>
                  <affiliation ref="#struct-559909"/>
                </author>
              </analytic>
              <monogr>
                <meeting>
                  <title>Workshop NeurIPS 2020 Human in the Loop Dialogue Systems</title>
                  <date type="start">2020-12-11</date>
                  <date type="end">2020-12-11</date>
                  <settlement>Virtual only</settlement>
                  <country key="US">United States</country>
                </meeting>
                <imprint/>
              </monogr>
              <idno type="arxiv">2102.13589</idno>
              <ref type="publisher">https://sites.google.com/view/hlds-2020/home?authuser=0</ref>
            </biblStruct>
          </sourceDesc>
          <profileDesc>
            <langUsage>
              <language ident="en">English</language>
            </langUsage>
            <textClass>
              <keywords scheme="author">
                <term xml:lang="en">autonomous learning</term>
                <term xml:lang="en">interactive learning</term>
                <term xml:lang="en">lifelong learning</term>
                <term xml:lang="en">on-the-job learning</term>
                <term xml:lang="en">task oriented dialogue systems</term>
              </keywords>
              <classCode scheme="halDomain" n="info.info-cl">Computer Science [cs]/Computation and Language [cs.CL]</classCode>
              <classCode scheme="halTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halOldTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halTreeTypology" n="COMM">Conference papers</classCode>
            </textClass>
            <abstract xml:lang="en">
              <p>On-the-job learning consists in continuously learning while being used in production, in an open environment, meaning that the system has to deal on its own with situations and elements never seen before. The kind of systems that seem to be especially adapted to on-the-job learning are dialogue systems, since they can take advantage of their interactions with users to collect feedback to adapt and improve their components over time. Some dialogue systems performing on-the-job learning have been built and evaluated but no general methodology has yet been defined. Thus in this paper, we propose a first general methodology for evaluating on-the-job learning dialogue systems. We also describe a task-oriented dialogue system which improves on-the-job its natural language component through its user interactions. We finally evaluate our system with the described methodology.</p>
            </abstract>
          </profileDesc>
        </biblFull>
      </listBibl>
    </body>
    <back>
      <listOrg type="structures">
        <org type="institution" xml:id="struct-559909" status="VALID">
          <idno type="ROR">https://ror.org/01ph39d13</idno>
          <orgName>Laboratoire National de Métrologie et d’Essais</orgName>
          <orgName type="acronym">LNE</orgName>
          <desc>
            <address>
              <addrLine>1, rue Gaston Boissier 75724 PARIS Cedex 15</addrLine>
              <country key="FR"/>
            </address>
          </desc>
        </org>
        <org type="researchteam" xml:id="struct-1041998" status="OLD">
          <orgName>Information, Langue Ecrite et Signée</orgName>
          <orgName type="acronym">ILES</orgName>
          <date type="start">2020-01-01</date>
          <date type="end">2020-12-31</date>
          <desc>
            <address>
              <addrLine>Bâtiment 508, 91405, Orsay cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.limsi.fr/fr/recherche/iles</ref>
          </desc>
          <listRelation>
            <relation active="#struct-1041968" type="direct"/>
            <relation active="#struct-419361" type="indirect"/>
            <relation name="UPR3251" active="#struct-441569" type="indirect"/>
          </listRelation>
        </org>
        <org type="laboratory" xml:id="struct-1041968" status="OLD">
          <orgName>Laboratoire d'Informatique pour la Mécanique et les Sciences de l'Ingénieur</orgName>
          <orgName type="acronym">LIMSI</orgName>
          <date type="start">2020-01-01</date>
          <date type="end">2020-12-31</date>
          <desc>
            <address>
              <addrLine>Bâtiment 507, rue du Belvédère,91405, Orsay cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.limsi.fr/fr/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-419361" type="direct"/>
            <relation name="UPR3251" active="#struct-441569" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-419361" status="VALID">
          <idno type="IdRef">241345251</idno>
          <idno type="ROR">https://ror.org/03xjwb503</idno>
          <orgName>Université Paris-Saclay</orgName>
          <desc>
            <address>
              <addrLine>Bâtiment Bréguet, 3 Rue Joliot Curie 2e ét, 91190 Gif-sur-Yvette</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.universite-paris-saclay.fr/fr</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-441569" status="VALID">
          <idno type="IdRef">02636817X</idno>
          <idno type="ISNI">0000000122597504</idno>
          <idno type="ROR">https://ror.org/02feahw73</idno>
          <orgName>Centre National de la Recherche Scientifique</orgName>
          <orgName type="acronym">CNRS</orgName>
          <date type="start">1939-10-19</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.cnrs.fr/</ref>
          </desc>
        </org>
      </listOrg>
    </back>
  </text>
</TEI>