<?xml version="1.0" encoding="utf-8"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hal="http://hal.archives-ouvertes.fr/" xmlns:gml="http://www.opengis.net/gml/3.3/" xmlns:gmlce="http://www.opengis.net/gml/3.3/ce" version="1.1" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 http://api.archives-ouvertes.fr/documents/aofr-sword.xsd">
  <teiHeader>
    <fileDesc>
      <titleStmt>
        <title>HAL TEI export of hal-04271020</title>
      </titleStmt>
      <publicationStmt>
        <distributor>CCSD</distributor>
        <availability status="restricted">
          <licence target="https://creativecommons.org/publicdomain/zero/1.0/">CC0 1.0 - Universal</licence>
        </availability>
        <date when="2026-05-15T22:08:12+02:00"/>
      </publicationStmt>
      <sourceDesc>
        <p part="N">HAL API Platform</p>
      </sourceDesc>
    </fileDesc>
  </teiHeader>
  <text>
    <body>
      <listBibl>
        <biblFull>
          <titleStmt>
            <title xml:lang="en">Approximate Heavy Tails in Offline (Multi-Pass) Stochastic Gradient Descent</title>
            <author role="aut">
              <persName>
                <forename type="first">Krunoslav Lehman</forename>
                <surname>Pavasovic</surname>
              </persName>
              <idno type="halauthorid">2942108-0</idno>
              <affiliation ref="#struct-454688"/>
              <affiliation ref="#struct-523723"/>
              <affiliation ref="#struct-441569"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Alain</forename>
                <surname>Durmus</surname>
              </persName>
              <email type="md5">01787fe59d16e0fb55fc0b40a5c095d7</email>
              <email type="domain">gmail.com</email>
              <idno type="idhal" notation="string">alain-oliviero-durmus</idno>
              <idno type="idhal" notation="numeric">1084098</idno>
              <idno type="halauthorid" notation="string">735110-1084098</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-2086-8611</idno>
              <idno type="IDREF">https://www.idref.fr/200536133</idno>
              <affiliation ref="#struct-89626"/>
              <affiliation ref="#struct-441569"/>
              <affiliation ref="#struct-563936"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Umut</forename>
                <surname>Simsekli</surname>
              </persName>
              <idno type="halauthorid">2693087-0</idno>
              <affiliation ref="#struct-441569"/>
              <affiliation ref="#struct-523723"/>
            </author>
            <editor role="depositor">
              <persName>
                <forename>Krunoslav</forename>
                <surname>Lehman Pavasovic</surname>
              </persName>
              <email type="md5">3e91971eead62df26a7810a2675eb159</email>
              <email type="domain">gmail.com</email>
            </editor>
            <funder ref="#projanr-50388"/>
            <funder ref="#projeurop-723128"/>
          </titleStmt>
          <editionStmt>
            <edition n="v1" type="current">
              <date type="whenSubmitted">2023-11-05 18:38:35</date>
              <date type="whenModified">2025-10-24 17:30:11</date>
              <date type="whenReleased">2023-11-05 18:38:35</date>
              <date type="whenProduced">2023-12-10</date>
              <ref type="externalLink" target="http://arxiv.org/pdf/2310.18455"/>
            </edition>
            <respStmt>
              <resp>contributor</resp>
              <name key="1469237">
                <persName>
                  <forename>Krunoslav</forename>
                  <surname>Lehman Pavasovic</surname>
                </persName>
                <email type="md5">3e91971eead62df26a7810a2675eb159</email>
                <email type="domain">gmail.com</email>
              </name>
            </respStmt>
          </editionStmt>
          <publicationStmt>
            <distributor>CCSD</distributor>
            <idno type="halId">hal-04271020</idno>
            <idno type="halUri">https://hal.science/hal-04271020</idno>
            <idno type="halBibtex">pavasovic:hal-04271020</idno>
            <idno type="halRefHtml">&lt;i&gt;Neural Information Processing Systems (NeurIPS), Spotlight Presentation, 2023&lt;/i&gt;, Dec 2023, New Orleans (LA), United States</idno>
            <idno type="halRef">Neural Information Processing Systems (NeurIPS), Spotlight Presentation, 2023, Dec 2023, New Orleans (LA), United States</idno>
            <availability status="restricted"/>
          </publicationStmt>
          <seriesStmt>
            <idno type="stamp" n="X">École polytechnique</idno>
            <idno type="stamp" n="ENS-PARIS">Ecole Normale Supérieure de Paris</idno>
            <idno type="stamp" n="CNRS">CNRS - Centre national de la recherche scientifique</idno>
            <idno type="stamp" n="INRIA">INRIA - Institut National de Recherche en Informatique et en Automatique</idno>
            <idno type="stamp" n="ENS-CACHAN">Ecole Normale Supérieure de Cachan</idno>
            <idno type="stamp" n="INRIA-ROCQ">INRIA Paris - Rocquencourt</idno>
            <idno type="stamp" n="INSMI">CNRS-INSMI - INstitut des Sciences Mathématiques et de leurs Interactions</idno>
            <idno type="stamp" n="X-CMAP" corresp="X">Centre de Mathématiques Appliquées (CMAP)</idno>
            <idno type="stamp" n="X-DEP-MATHA">Département de mathématiques appliquées de l’École polytechnique</idno>
            <idno type="stamp" n="OPENAIRE">OpenAIRE</idno>
            <idno type="stamp" n="TESTALAIN1">TESTALAIN1</idno>
            <idno type="stamp" n="CMAP">Centre de Mathématiques Appliquées (CMAP)</idno>
            <idno type="stamp" n="INRIA2">INRIA 2</idno>
            <idno type="stamp" n="PSL">Université Paris sciences et lettres</idno>
            <idno type="stamp" n="INRIA-PSL">INRIA-PSL</idno>
            <idno type="stamp" n="IP_PARIS">Institut Polytechnique de Paris</idno>
            <idno type="stamp" n="PNRIA">Programme National de Recherche en IA</idno>
            <idno type="stamp" n="ENS-PSL" corresp="PSL">École normale supérieure - PSL</idno>
            <idno type="stamp" n="ANR">ANR</idno>
            <idno type="stamp" n="PRAIRIE-IA">PaRis AI Research InstitutE</idno>
            <idno type="stamp" n="ENS-PARIS-SACLAY">Ecole Normale Supérieure paris-Saclay</idno>
            <idno type="stamp" n="DIENS" corresp="ENS-PARIS">Département d'informatique de l'ENS-PSL</idno>
            <idno type="stamp" n="ANR-IA-19" corresp="ANR-IA">ANR-IA-19</idno>
            <idno type="stamp" n="ANR-IA">ANR-IA</idno>
            <idno type="stamp" n="DEPARTEMENT-DE-MATHEMATIQUES">Collection du Département de Mathématiques</idno>
          </seriesStmt>
          <notesStmt>
            <note type="commentary">In Neural Information Processing Systems (NeurIPS), Spotlight Presentation, 2023</note>
            <note type="audience" n="2">International</note>
            <note type="invited" n="0">No</note>
            <note type="popular" n="0">No</note>
            <note type="peer" n="1">Yes</note>
            <note type="proceedings" n="1">Yes</note>
          </notesStmt>
          <sourceDesc>
            <biblStruct>
              <analytic>
                <title xml:lang="en">Approximate Heavy Tails in Offline (Multi-Pass) Stochastic Gradient Descent</title>
                <author role="aut">
                  <persName>
                    <forename type="first">Krunoslav Lehman</forename>
                    <surname>Pavasovic</surname>
                  </persName>
                  <idno type="halauthorid">2942108-0</idno>
                  <affiliation ref="#struct-454688"/>
                  <affiliation ref="#struct-523723"/>
                  <affiliation ref="#struct-441569"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Alain</forename>
                    <surname>Durmus</surname>
                  </persName>
                  <email type="md5">01787fe59d16e0fb55fc0b40a5c095d7</email>
                  <email type="domain">gmail.com</email>
                  <idno type="idhal" notation="string">alain-oliviero-durmus</idno>
                  <idno type="idhal" notation="numeric">1084098</idno>
                  <idno type="halauthorid" notation="string">735110-1084098</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-2086-8611</idno>
                  <idno type="IDREF">https://www.idref.fr/200536133</idno>
                  <affiliation ref="#struct-89626"/>
                  <affiliation ref="#struct-441569"/>
                  <affiliation ref="#struct-563936"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Umut</forename>
                    <surname>Simsekli</surname>
                  </persName>
                  <idno type="halauthorid">2693087-0</idno>
                  <affiliation ref="#struct-441569"/>
                  <affiliation ref="#struct-523723"/>
                </author>
              </analytic>
              <monogr>
                <meeting>
                  <title>Neural Information Processing Systems (NeurIPS), Spotlight Presentation, 2023</title>
                  <date type="start">2023-12-10</date>
                  <date type="end">2023-12-16</date>
                  <settlement>New Orleans (LA)</settlement>
                  <country key="US">United States</country>
                </meeting>
                <imprint>
                  <date type="datePub">2023-10-27</date>
                </imprint>
              </monogr>
              <idno type="arxiv">2310.18455</idno>
            </biblStruct>
          </sourceDesc>
          <profileDesc>
            <langUsage>
              <language ident="en">English</language>
            </langUsage>
            <textClass>
              <keywords scheme="author">
                <term xml:lang="en">Heavy tails</term>
                <term xml:lang="en">Optimization</term>
                <term xml:lang="en">Stochastic gradient descent</term>
                <term xml:lang="en">Heavy tails</term>
              </keywords>
              <classCode scheme="halDomain" n="info">Computer Science [cs]</classCode>
              <classCode scheme="halDomain" n="math">Mathematics [math]</classCode>
              <classCode scheme="halDomain" n="stat">Statistics [stat]</classCode>
              <classCode scheme="halTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halOldTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halTreeTypology" n="COMM">Conference papers</classCode>
            </textClass>
            <abstract xml:lang="en">
              <p>A recent line of empirical studies has demonstrated that SGD might exhibit a heavy-tailed behavior in practical settings, and the heaviness of the tails might correlate with the overall performance. In this paper, we investigate the emergence of such heavy tails. Previous works on this problem only considered, up to our knowledge, online (also called single-pass) SGD, in which the emergence of heavy tails in theoretical findings is contingent upon access to an infinite amount of data. Hence, the underlying mechanism generating the reported heavy-tailed behavior in practical settings, where the amount of training data is finite, is still not well-understood. Our contribution aims to fill this gap. In particular, we show that the stationary distribution of offline (also called multi-pass) SGD exhibits 'approximate' power-law tails and the approximation error is controlled by how fast the empirical distribution of the training data converges to the true underlying data distribution in the Wasserstein metric. Our main takeaway is that, as the number of data points increases, offline SGD will behave increasingly 'power-law-like'. To achieve this result, we first prove nonasymptotic Wasserstein convergence bounds for offline SGD to online SGD as the number of data points increases, which can be interesting on their own. Finally, we illustrate our theory on various experiments conducted on synthetic data and neural networks.</p>
            </abstract>
          </profileDesc>
        </biblFull>
      </listBibl>
    </body>
    <back>
      <listOrg type="structures">
        <org type="researchteam" xml:id="struct-454688" status="VALID">
          <idno type="IdRef">24507595X</idno>
          <idno type="RNSR">201120973D</idno>
          <idno type="ROR">https://ror.org/0485z7y90</idno>
          <orgName>Statistical Machine Learning and Parsimony</orgName>
          <orgName type="acronym">SIERRA</orgName>
          <date type="start">2016-01-01</date>
          <date type="end">2028-12-31</date>
          <desc>
            <address>
              <addrLine>48 Rue Barrault, 75013 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/equipes/sierra</ref>
          </desc>
          <listRelation>
            <relation active="#struct-25027" type="direct"/>
            <relation active="#struct-59704" type="indirect"/>
            <relation active="#struct-564132" type="indirect"/>
            <relation active="#struct-300009" type="indirect"/>
            <relation name="UMR8548" active="#struct-441569" type="direct"/>
            <relation active="#struct-441569" type="direct"/>
            <relation active="#struct-454310" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-523723" status="VALID">
          <idno type="ROR">https://ror.org/00hx6zz33</idno>
          <orgName>Ecole Normale Supérieure  Paris-Saclay</orgName>
          <orgName type="acronym">ENS Paris Saclay</orgName>
          <desc>
            <address>
              <addrLine>4 avenue des Sciences, 91190 Gif-sur-Yvette</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://ens-paris-saclay.fr/</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-441569" status="VALID">
          <idno type="IdRef">02636817X</idno>
          <idno type="ISNI">0000000122597504</idno>
          <idno type="ROR">https://ror.org/02feahw73</idno>
          <orgName>Centre National de la Recherche Scientifique</orgName>
          <orgName type="acronym">CNRS</orgName>
          <date type="start">1939-10-19</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.cnrs.fr/</ref>
          </desc>
        </org>
        <org type="laboratory" xml:id="struct-89626" status="VALID">
          <idno type="IdRef">197373461</idno>
          <idno type="ISNI">000000010944436X</idno>
          <idno type="RNSR">199719340P</idno>
          <idno type="ROR">https://ror.org/012e1xn46</idno>
          <idno type="Wikidata">Q16008922</idno>
          <orgName>Centre de Mathématiques Appliquées de l'Ecole polytechnique</orgName>
          <orgName type="acronym">CMAP</orgName>
          <date type="start">2005-01-01</date>
          <desc>
            <address>
              <addrLine>Route de Saclay, 91128 Palaiseau Cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://cmap.ip-paris.fr/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-300009" type="direct"/>
            <relation active="#struct-300340" type="direct"/>
            <relation active="#struct-563936" type="indirect"/>
            <relation name="UMR7641" active="#struct-441569" type="direct"/>
          </listRelation>
        </org>
        <org type="regroupinstitution" xml:id="struct-563936" status="VALID">
          <idno type="IdRef">238327159</idno>
          <idno type="ISNI">0000000502717600</idno>
          <idno type="ROR">https://ror.org/042tfbd02</idno>
          <idno type="Wikidata">Q48759778</idno>
          <orgName>Institut Polytechnique de Paris</orgName>
          <orgName type="acronym">IP Paris</orgName>
          <date type="start">2019-06-02</date>
          <desc>
            <address>
              <addrLine>Route de Saclay, 91120 Palaiseau Cedex, France</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.ip-paris.fr</ref>
          </desc>
        </org>
        <org type="regrouplaboratory" xml:id="struct-25027" status="VALID">
          <idno type="IdRef">148034055</idno>
          <idno type="RNSR">199812876J</idno>
          <idno type="ROR">05y6rqs46</idno>
          <orgName>Département d'informatique - ENS-PSL</orgName>
          <orgName type="acronym">DI-ENS</orgName>
          <date type="start">1999-01-01</date>
          <desc>
            <address>
              <addrLine>École normale supérieure 45 rue d'Ulm F-75230 Paris Cedex 05</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.di.ens.fr/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-59704" type="direct"/>
            <relation active="#struct-564132" type="indirect"/>
            <relation active="#struct-300009" type="direct"/>
            <relation name="UMR8548" active="#struct-441569" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-59704" status="VALID">
          <idno type="IdRef">031738419</idno>
          <idno type="ISNI">0000000123532622</idno>
          <idno type="ROR">https://ror.org/05a0dhs15</idno>
          <orgName>École normale supérieure - Paris</orgName>
          <orgName type="acronym">ENS-PSL</orgName>
          <date type="start">1985-07-24</date>
          <desc>
            <address>
              <addrLine>45, Rue d'Ulm - 75230 Paris cedex 05</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.ens.psl.eu/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-564132" type="direct"/>
          </listRelation>
        </org>
        <org type="regroupinstitution" xml:id="struct-564132" status="VALID">
          <idno type="IdRef">241597595</idno>
          <idno type="ISNI">0000 0004 1784 3645</idno>
          <idno type="ROR">https://ror.org/013cjyk83</idno>
          <orgName>Université Paris Sciences et Lettres</orgName>
          <orgName type="acronym">PSL</orgName>
          <desc>
            <address>
              <addrLine>60 rue Mazarine 75006 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.psl.eu/</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-300009" status="VALID">
          <idno type="ROR">https://ror.org/02kvxyf05</idno>
          <orgName>Institut National de Recherche en Informatique et en Automatique</orgName>
          <orgName type="acronym">Inria</orgName>
          <desc>
            <address>
              <addrLine>Domaine de VoluceauRocquencourt - BP 10578153 Le Chesnay Cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/en/</ref>
          </desc>
        </org>
        <org type="laboratory" xml:id="struct-454310" status="VALID">
          <idno type="IdRef">241614864</idno>
          <idno type="RNSR">196718247G</idno>
          <idno type="ROR">https://ror.org/05eyd5d35</idno>
          <orgName>Centre Inria de Paris</orgName>
          <date type="start">2016-03-10</date>
          <desc>
            <address>
              <addrLine>48 Rue Barrault, 75013 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/centre/paris</ref>
          </desc>
          <listRelation>
            <relation active="#struct-300009" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-300340" status="VALID">
          <idno type="IdRef">027309320</idno>
          <idno type="ISNI">0000000121581279</idno>
          <idno type="ROR">https://ror.org/05hy3tk52</idno>
          <idno type="Wikidata">Q273626</idno>
          <orgName>École polytechnique</orgName>
          <orgName type="acronym">X</orgName>
          <date type="start">1794-03-11</date>
          <desc>
            <address>
              <addrLine>Route de Saclay, 91128 Palaiseau Cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.polytechnique.edu/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-563936" type="direct"/>
          </listRelation>
        </org>
      </listOrg>
      <listOrg type="projects">
        <org type="anrProject" xml:id="projanr-50388" status="VALID">
          <idno type="anr">ANR-19-P3IA-0001</idno>
          <orgName>PRAIRIE</orgName>
          <desc>PaRis Artificial Intelligence Research InstitutE</desc>
          <date type="start">2019</date>
        </org>
        <org type="europeanProject" xml:id="projeurop-723128" status="VALID">
          <idno type="number">101039676</idno>
          <idno type="program">ERC-2021-STG</idno>
          <idno type="call">ERC-2021-STG</idno>
          <orgName>DYNASTY</orgName>
          <desc>Dynamics-Aware Theory of Deep Learning</desc>
          <date type="start">2022-10-01</date>
          <date type="end">2027-09-30</date>
        </org>
      </listOrg>
    </back>
  </text>
</TEI>