<?xml version="1.0" encoding="utf-8"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hal="http://hal.archives-ouvertes.fr/" xmlns:gml="http://www.opengis.net/gml/3.3/" xmlns:gmlce="http://www.opengis.net/gml/3.3/ce" version="1.1" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 http://api.archives-ouvertes.fr/documents/aofr-sword.xsd">
  <teiHeader>
    <fileDesc>
      <titleStmt>
        <title>HAL TEI export of hal-01968558</title>
      </titleStmt>
      <publicationStmt>
        <distributor>CCSD</distributor>
        <availability status="restricted">
          <licence target="https://creativecommons.org/publicdomain/zero/1.0/">CC0 1.0 - Universal</licence>
        </availability>
        <date when="2026-05-22T22:35:39+02:00"/>
      </publicationStmt>
      <sourceDesc>
        <p part="N">HAL API Platform</p>
      </sourceDesc>
    </fileDesc>
  </teiHeader>
  <text>
    <body>
      <listBibl>
        <biblFull>
          <titleStmt>
            <title xml:lang="en">On Matrix Momentum Stochastic Approximation and Applications to Q-learning</title>
            <author role="aut">
              <persName>
                <forename type="first">Adithya M.</forename>
                <surname>Devraj</surname>
              </persName>
              <idno type="halauthorid">1491001-0</idno>
              <affiliation ref="#struct-42900"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Ana</forename>
                <surname>Bušić</surname>
              </persName>
              <email type="md5">4468098cbb90d6145cb78d60c7848e4d</email>
              <email type="domain">liafa.jussieu.fr</email>
              <idno type="idhal" notation="string">anabusic</idno>
              <idno type="idhal" notation="numeric">2602</idno>
              <idno type="halauthorid" notation="string">20194-2602</idno>
              <idno type="IDREF">https://www.idref.fr/144488175</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-4133-3739</idno>
              <affiliation ref="#struct-454323"/>
              <affiliation ref="#struct-160294"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Sean</forename>
                <surname>Meyn</surname>
              </persName>
              <idno type="halauthorid">155854-0</idno>
              <affiliation ref="#struct-42900"/>
            </author>
            <editor role="depositor">
              <persName>
                <forename>Ana</forename>
                <surname>Busic</surname>
              </persName>
              <email type="md5">6a52e380114c103a33b2ed620abfa69b</email>
              <email type="domain">inria.fr</email>
            </editor>
            <funder ref="#projanr-42416"/>
          </titleStmt>
          <editionStmt>
            <edition n="v1" type="current">
              <date type="whenSubmitted">2019-01-02 19:34:23</date>
              <date type="whenWritten">2018-09-17</date>
              <date type="whenModified">2026-02-07 05:15:48</date>
              <date type="whenReleased">2019-01-02 19:34:23</date>
              <date type="whenProduced">2019-09-24</date>
              <ref type="externalLink" target="http://arxiv.org/pdf/1809.06277"/>
            </edition>
            <respStmt>
              <resp>contributor</resp>
              <name key="137349">
                <persName>
                  <forename>Ana</forename>
                  <surname>Busic</surname>
                </persName>
                <email type="md5">6a52e380114c103a33b2ed620abfa69b</email>
                <email type="domain">inria.fr</email>
              </name>
            </respStmt>
          </editionStmt>
          <publicationStmt>
            <distributor>CCSD</distributor>
            <idno type="halId">hal-01968558</idno>
            <idno type="halUri">https://hal.science/hal-01968558</idno>
            <idno type="halBibtex">devraj:hal-01968558</idno>
            <idno type="halRefHtml">&lt;i&gt;57th Annual Allerton Conference on Communication, Control, and Computing (Allerton)&lt;/i&gt;, Sep 2019, Monticello, IL, United States. &lt;a target="_blank" href="https://dx.doi.org/10.1109/ALLERTON.2019.8919828"&gt;&amp;#x27E8;10.1109/ALLERTON.2019.8919828&amp;#x27E9;&lt;/a&gt;</idno>
            <idno type="halRef">57th Annual Allerton Conference on Communication, Control, and Computing (Allerton), Sep 2019, Monticello, IL, United States. &amp;#x27E8;10.1109/ALLERTON.2019.8919828&amp;#x27E9;</idno>
            <availability status="restricted"/>
          </publicationStmt>
          <seriesStmt>
            <idno type="stamp" n="ENS-PARIS">Ecole Normale Supérieure de Paris</idno>
            <idno type="stamp" n="CNRS">CNRS - Centre national de la recherche scientifique</idno>
            <idno type="stamp" n="INRIA">INRIA - Institut National de Recherche en Informatique et en Automatique</idno>
            <idno type="stamp" n="INRIA-ROCQ">INRIA Paris - Rocquencourt</idno>
            <idno type="stamp" n="TESTALAIN1">TESTALAIN1</idno>
            <idno type="stamp" n="INRIA2">INRIA 2</idno>
            <idno type="stamp" n="TDS-MACS">Réseau de recherche en Théorie des Systèmes Distribués, Modélisation, Analyse et Contrôle des Systèmes</idno>
            <idno type="stamp" n="PSL">Université Paris sciences et lettres</idno>
            <idno type="stamp" n="INRIA-PSL">INRIA-PSL</idno>
            <idno type="stamp" n="SORBONNE-UNIVERSITE">Sorbonne Université</idno>
            <idno type="stamp" n="SORBONNE-UNIV" corresp="SORBONNE-UNIVERSITE">Sorbonne Université 01/01/2018</idno>
            <idno type="stamp" n="SU-SCIENCES" corresp="SORBONNE-UNIVERSITE">Faculté des Sciences de Sorbonne Université</idno>
            <idno type="stamp" n="SU-SCI" corresp="SORBONNE-UNIVERSITE">Sciences - Sorbonne Université</idno>
            <idno type="stamp" n="INSTITUTS-TELECOM">composantes instituts telecom </idno>
            <idno type="stamp" n="ENS-PSL" corresp="PSL">École normale supérieure - PSL</idno>
            <idno type="stamp" n="SU-TI">Sorbonne Université - Texte Intégral</idno>
            <idno type="stamp" n="ANR">ANR</idno>
            <idno type="stamp" n="ALLIANCE-SU"> Alliance Sorbonne Université</idno>
            <idno type="stamp" n="INRIA-ETATSUNIS">Copublications Inria-Etats-Unis</idno>
            <idno type="stamp" n="DIENS" corresp="ENS-PARIS">Département d'informatique de l'ENS-PSL</idno>
          </seriesStmt>
          <notesStmt>
            <note type="audience" n="2">International</note>
            <note type="invited" n="0">No</note>
            <note type="popular" n="0">No</note>
            <note type="peer" n="1">Yes</note>
            <note type="proceedings" n="1">Yes</note>
          </notesStmt>
          <sourceDesc>
            <biblStruct>
              <analytic>
                <title xml:lang="en">On Matrix Momentum Stochastic Approximation and Applications to Q-learning</title>
                <author role="aut">
                  <persName>
                    <forename type="first">Adithya M.</forename>
                    <surname>Devraj</surname>
                  </persName>
                  <idno type="halauthorid">1491001-0</idno>
                  <affiliation ref="#struct-42900"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Ana</forename>
                    <surname>Bušić</surname>
                  </persName>
                  <email type="md5">4468098cbb90d6145cb78d60c7848e4d</email>
                  <email type="domain">liafa.jussieu.fr</email>
                  <idno type="idhal" notation="string">anabusic</idno>
                  <idno type="idhal" notation="numeric">2602</idno>
                  <idno type="halauthorid" notation="string">20194-2602</idno>
                  <idno type="IDREF">https://www.idref.fr/144488175</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-4133-3739</idno>
                  <affiliation ref="#struct-454323"/>
                  <affiliation ref="#struct-160294"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Sean</forename>
                    <surname>Meyn</surname>
                  </persName>
                  <idno type="halauthorid">155854-0</idno>
                  <affiliation ref="#struct-42900"/>
                </author>
              </analytic>
              <monogr>
                <meeting>
                  <title>57th Annual Allerton Conference on Communication, Control, and Computing (Allerton)</title>
                  <date type="start">2019-09-24</date>
                  <date type="end">2019-09-27</date>
                  <settlement>Monticello, IL</settlement>
                  <country key="US">United States</country>
                </meeting>
                <imprint>
                  <biblScope unit="serie">Proceedings of 57th Annual Allerton Conference on Communication, Control, and Computing (Allerton)</biblScope>
                  <date type="datePub">2019</date>
                </imprint>
              </monogr>
              <idno type="arxiv">1809.06277</idno>
              <idno type="doi">10.1109/ALLERTON.2019.8919828</idno>
            </biblStruct>
          </sourceDesc>
          <profileDesc>
            <langUsage>
              <language ident="en">English</language>
            </langUsage>
            <textClass>
              <classCode scheme="halDomain" n="math.math-oc">Mathematics [math]/Optimization and Control [math.OC]</classCode>
              <classCode scheme="halDomain" n="info.info-lg">Computer Science [cs]/Machine Learning [cs.LG]</classCode>
              <classCode scheme="halDomain" n="math.math-pr">Mathematics [math]/Probability [math.PR]</classCode>
              <classCode scheme="halDomain" n="info.info-ai">Computer Science [cs]/Artificial Intelligence [cs.AI]</classCode>
              <classCode scheme="halTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halOldTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halTreeTypology" n="COMM">Conference papers</classCode>
            </textClass>
            <abstract xml:lang="en">
              <p>Stochastic approximation (SA) algorithms are recursive techniques used to obtain the roots of functions that can be expressed as expectations of a noisy parameterized family of functions. In this paper two new SA algorithms are introduced: 1) PolSA, an extension of Polyak’s momentum technique with a specially designed matrix momentum, and 2) NeSA, which can either be regarded as a variant of Nesterov’s acceleration method, or a simplification of PolSA. The rates of convergence of SA algorithms is well understood. Under special conditions, the mean square error of the parameter estimates is bounded by $\sigma^{2}/n+o(1/n)$, where $\sigma^{2} \geq 0$ is an identifiable constant. If these conditions fail, the rate is typically sub-linear. There are two well known SA algorithms that ensure a linear rate, with minimal value of variance, $\sigma^{2}$: the Ruppert-Polyak averaging technique, and the stochastic Newton-Raphson (SNR) algorithm. It is demonstrated here that under mild technical assumptions, the PolSA algorithm also achieves this optimality criteria. This result is established via novel coupling arguments: It is shown that the parameter estimates obtained from the PolSA algorithm couple with those of the optimal variance (but computationally more expensive) SNR algorithm, at a rate $O(1/n^{2})$. The newly proposed algorithms are extended to a reinforcement learning setting to obtain new Q-learning algorithms, and numerical results confirm the coupling of PolSA and SNR.</p>
            </abstract>
          </profileDesc>
        </biblFull>
      </listBibl>
    </body>
    <back>
      <listOrg type="structures">
        <org type="regrouplaboratory" xml:id="struct-42900" status="VALID">
          <orgName>Department of Electrical and Computer Engineering [Gainesville]</orgName>
          <orgName type="acronym">UF|ECE</orgName>
          <desc>
            <address>
              <addrLine>University of Florida, 216 Larsen Hall Gainesville, FL 32611-6200</addrLine>
              <country key="US"/>
            </address>
            <ref type="url">https://www.ece.ufl.edu/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-302308" type="direct"/>
          </listRelation>
        </org>
        <org type="researchteam" xml:id="struct-454323" status="OLD">
          <idno type="IdRef">242745202</idno>
          <idno type="RNSR">201321099G</idno>
          <idno type="ROR">https://ror.org/02njc8q57</idno>
          <orgName>Dynamics of Geometric Networks</orgName>
          <orgName type="acronym">DYOGENE</orgName>
          <date type="start">2016-03-31</date>
          <date type="end">2025-05-31</date>
          <desc>
            <address>
              <addrLine>48 Rue Barrault, 75013 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/equipes/dyogene</ref>
          </desc>
          <listRelation>
            <relation active="#struct-25027" type="direct"/>
            <relation active="#struct-59704" type="indirect"/>
            <relation active="#struct-564132" type="indirect"/>
            <relation active="#struct-300009" type="indirect"/>
            <relation name="UMR8548" active="#struct-441569" type="indirect"/>
            <relation active="#struct-454310" type="direct"/>
          </listRelation>
        </org>
        <org type="laboratory" xml:id="struct-160294" status="OLD">
          <orgName>Laboratory of Information, Network and Communication Sciences</orgName>
          <orgName type="acronym">LINCS</orgName>
          <desc>
            <address>
              <addrLine>23 avenue d'Italie 75013 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.lincs.fr/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-93591" type="direct"/>
            <relation active="#struct-300009" type="direct"/>
            <relation active="#struct-302102" type="direct"/>
          </listRelation>
        </org>
        <org type="regroupinstitution" xml:id="struct-302308" status="VALID">
          <idno type="ROR">https://ror.org/02y3ad647</idno>
          <orgName>University of Florida [Gainesville]</orgName>
          <orgName type="acronym">UF</orgName>
          <desc>
            <address>
              <addrLine>Gainesville, FL 32611</addrLine>
              <country key="US"/>
            </address>
            <ref type="url">http://www.ufl.edu/</ref>
          </desc>
        </org>
        <org type="regrouplaboratory" xml:id="struct-25027" status="VALID">
          <idno type="IdRef">148034055</idno>
          <idno type="RNSR">199812876J</idno>
          <idno type="ROR">05y6rqs46</idno>
          <orgName>Département d'informatique - ENS-PSL</orgName>
          <orgName type="acronym">DI-ENS</orgName>
          <date type="start">1999-01-01</date>
          <desc>
            <address>
              <addrLine>École normale supérieure 45 rue d'Ulm F-75230 Paris Cedex 05</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.di.ens.fr/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-59704" type="direct"/>
            <relation active="#struct-564132" type="indirect"/>
            <relation active="#struct-300009" type="direct"/>
            <relation name="UMR8548" active="#struct-441569" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-59704" status="VALID">
          <idno type="IdRef">031738419</idno>
          <idno type="ISNI">0000000123532622</idno>
          <idno type="ROR">https://ror.org/05a0dhs15</idno>
          <orgName>École normale supérieure - Paris</orgName>
          <orgName type="acronym">ENS-PSL</orgName>
          <date type="start">1985-07-24</date>
          <desc>
            <address>
              <addrLine>45, Rue d'Ulm - 75230 Paris cedex 05</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.ens.psl.eu/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-564132" type="direct"/>
          </listRelation>
        </org>
        <org type="regroupinstitution" xml:id="struct-564132" status="VALID">
          <idno type="IdRef">241597595</idno>
          <idno type="ISNI">0000 0004 1784 3645</idno>
          <idno type="ROR">https://ror.org/013cjyk83</idno>
          <orgName>Université Paris Sciences et Lettres</orgName>
          <orgName type="acronym">PSL</orgName>
          <desc>
            <address>
              <addrLine>60 rue Mazarine 75006 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.psl.eu/</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-300009" status="VALID">
          <idno type="ROR">https://ror.org/02kvxyf05</idno>
          <orgName>Institut National de Recherche en Informatique et en Automatique</orgName>
          <orgName type="acronym">Inria</orgName>
          <desc>
            <address>
              <addrLine>Domaine de VoluceauRocquencourt - BP 10578153 Le Chesnay Cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/en/</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-441569" status="VALID">
          <idno type="IdRef">02636817X</idno>
          <idno type="ISNI">0000000122597504</idno>
          <idno type="ROR">https://ror.org/02feahw73</idno>
          <orgName>Centre National de la Recherche Scientifique</orgName>
          <orgName type="acronym">CNRS</orgName>
          <date type="start">1939-10-19</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.cnrs.fr/</ref>
          </desc>
        </org>
        <org type="laboratory" xml:id="struct-454310" status="VALID">
          <idno type="IdRef">241614864</idno>
          <idno type="RNSR">196718247G</idno>
          <idno type="ROR">https://ror.org/05eyd5d35</idno>
          <orgName>Centre Inria de Paris</orgName>
          <date type="start">2016-03-10</date>
          <desc>
            <address>
              <addrLine>48 Rue Barrault, 75013 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/centre/paris</ref>
          </desc>
          <listRelation>
            <relation active="#struct-300009" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-93591" status="OLD">
          <idno type="ROR">https://ror.org/02en5vm52</idno>
          <orgName>Université Pierre et Marie Curie - Paris 6</orgName>
          <orgName type="acronym">UPMC</orgName>
          <date type="end">2017-12-31</date>
          <desc>
            <address>
              <addrLine>4 place Jussieu - 75005 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.upmc.fr/</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-302102" status="VALID">
          <idno type="IdRef">192427156</idno>
          <idno type="ISNI">000000012202567X</idno>
          <idno type="ROR">https://ror.org/025vp2923</idno>
          <idno type="Wikidata">Q27962533</idno>
          <orgName>Institut Mines-Télécom [Paris]</orgName>
          <orgName type="acronym">IMT</orgName>
          <date type="start">2012-03-01</date>
          <desc>
            <address>
              <addrLine>19 Place Marguerite Perey, 91120 Palaiseau</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.imt.fr/</ref>
          </desc>
        </org>
      </listOrg>
      <listOrg type="projects">
        <org type="anrProject" xml:id="projanr-42416" status="VALID">
          <idno type="anr">ANR-16-CE05-0008</idno>
          <orgName>PARI</orgName>
          <desc>Approche probabiliste pour l'intégration des énergies renouvelables : le stockage virtuel en utilisant la flexibilité de la demande</desc>
          <date type="start">2016</date>
        </org>
      </listOrg>
    </back>
  </text>
</TEI>