<?xml version="1.0" encoding="utf-8"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hal="http://hal.archives-ouvertes.fr/" xmlns:gml="http://www.opengis.net/gml/3.3/" xmlns:gmlce="http://www.opengis.net/gml/3.3/ce" version="1.1" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 http://api.archives-ouvertes.fr/documents/aofr-sword.xsd">
  <teiHeader>
    <fileDesc>
      <titleStmt>
        <title>HAL TEI export of hal-05578815</title>
      </titleStmt>
      <publicationStmt>
        <distributor>CCSD</distributor>
        <availability status="restricted">
          <licence target="https://creativecommons.org/publicdomain/zero/1.0/">CC0 1.0 - Universal</licence>
        </availability>
        <date when="2026-05-04T08:43:47+02:00"/>
      </publicationStmt>
      <sourceDesc>
        <p part="N">HAL API Platform</p>
      </sourceDesc>
    </fileDesc>
  </teiHeader>
  <text>
    <body>
      <listBibl>
        <biblFull>
          <titleStmt>
            <title xml:lang="en">Homogenized Transformers</title>
            <author role="aut">
              <persName>
                <forename type="first">Hugo</forename>
                <surname>Koubbi</surname>
              </persName>
              <idno type="halauthorid">3260600-0</idno>
              <affiliation ref="#struct-60"/>
              <affiliation ref="#struct-1303693"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Borjan</forename>
                <surname>Geshkovski</surname>
              </persName>
              <email type="md5">fc9f4eed97b14fe1624461ea234bab4f</email>
              <email type="domain">inria.fr</email>
              <idno type="idhal" notation="string">borjan-geshkovski</idno>
              <idno type="idhal" notation="numeric">1053249</idno>
              <idno type="halauthorid" notation="string">1384931-1053249</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-7890-3352</idno>
              <affiliation ref="#struct-1005052"/>
              <affiliation ref="#struct-1303693"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Philippe</forename>
                <surname>Rigollet</surname>
              </persName>
              <email type="md5">7f986290561726ed1e7864f93b4adb07</email>
              <email type="domain">mit.edu</email>
              <idno type="idhal" notation="numeric">1665765</idno>
              <idno type="halauthorid" notation="string">118324-1665765</idno>
              <affiliation ref="#struct-103056"/>
            </author>
            <editor role="depositor">
              <persName>
                <forename>Borjan</forename>
                <surname>Geshkovski</surname>
              </persName>
              <email type="md5">fc9f4eed97b14fe1624461ea234bab4f</email>
              <email type="domain">inria.fr</email>
            </editor>
            <funder>B.G. was supported by a Sorbonne Emergences grant, and a gift from Google. P.R. was supported by NSF grants DMS-2022448, CCF-2106377, and a gift from Apple</funder>
          </titleStmt>
          <editionStmt>
            <edition n="v1" type="current">
              <date type="whenSubmitted">2026-04-03 10:39:45</date>
              <date type="whenModified">2026-04-07 10:50:35</date>
              <date type="whenReleased">2026-04-03 11:05:00</date>
              <date type="whenProduced">2026-04-03</date>
              <date type="whenEndEmbargoed">2026-04-03</date>
              <ref type="file" target="https://hal.science/hal-05578815v1/document">
                <date notBefore="2026-04-03"/>
              </ref>
              <ref type="file" subtype="author" n="1" target="https://hal.science/hal-05578815v1/file/2604.01978v1.pdf" id="file-5578815-4778511">
                <date notBefore="2026-04-03"/>
              </ref>
              <ref type="externalLink" target="http://arxiv.org/pdf/2604.01978"/>
            </edition>
            <respStmt>
              <resp>contributor</resp>
              <name key="1556549">
                <persName>
                  <forename>Borjan</forename>
                  <surname>Geshkovski</surname>
                </persName>
                <email type="md5">fc9f4eed97b14fe1624461ea234bab4f</email>
                <email type="domain">inria.fr</email>
              </name>
            </respStmt>
          </editionStmt>
          <publicationStmt>
            <distributor>CCSD</distributor>
            <idno type="halId">hal-05578815</idno>
            <idno type="halUri">https://hal.science/hal-05578815</idno>
            <idno type="halBibtex">koubbi:hal-05578815</idno>
            <idno type="halRefHtml">2026</idno>
            <idno type="halRef">2026</idno>
            <availability status="restricted">
              <licence target="https://creativecommons.org/licenses/by-nc-nd/4.0/">CC BY-NC-ND 4.0 - Attribution - Non-commercial use - No Derivative Works<ref corresp="#file-5578815-4778511"/></licence>
            </availability>
          </publicationStmt>
          <seriesStmt>
            <idno type="stamp" n="CNRS">CNRS - Centre national de la recherche scientifique</idno>
            <idno type="stamp" n="INRIA">INRIA - Institut National de Recherche en Informatique et en Automatique</idno>
            <idno type="stamp" n="UNIV-DAUPHINE">Université Paris Dauphine - Paris IX</idno>
            <idno type="stamp" n="INRIA-ROCQ">INRIA Paris - Rocquencourt</idno>
            <idno type="stamp" n="INSMI">CNRS-INSMI - INstitut des Sciences Mathématiques et de leurs Interactions</idno>
            <idno type="stamp" n="CEREMADE" corresp="UNIV-DAUPHINE">CEntre de REcherches en MAthématiques de la DEcision</idno>
            <idno type="stamp" n="LJLL" corresp="SORBONNE-UNIVERSITE">Laboratoire Jacques-Louis Lions</idno>
            <idno type="stamp" n="TESTALAIN1">TESTALAIN1</idno>
            <idno type="stamp" n="INRIA2">INRIA 2</idno>
            <idno type="stamp" n="PSL">Université Paris sciences et lettres</idno>
            <idno type="stamp" n="LPSM" corresp="SORBONNE-UNIVERSITE">Laboratoire de Probabilités, Statistique et Modélisation</idno>
            <idno type="stamp" n="SORBONNE-UNIVERSITE">Sorbonne Université</idno>
            <idno type="stamp" n="SORBONNE-UNIV" corresp="SORBONNE-UNIVERSITE">Sorbonne Université 01/01/2018</idno>
            <idno type="stamp" n="SU-SCIENCES" corresp="SORBONNE-UNIVERSITE">Faculté des Sciences de Sorbonne Université</idno>
            <idno type="stamp" n="UNIV-PARIS">Université Paris Cité</idno>
            <idno type="stamp" n="UNIVERSITE-PARIS" corresp="UNIV-PARIS">Université Paris Cité</idno>
            <idno type="stamp" n="UP-SCIENCES">Université Paris Cité - Faculté des Sciences</idno>
            <idno type="stamp" n="UNIV-DAUPHINE-PSL" corresp="PSL">Université Paris Dauphine - PSL</idno>
            <idno type="stamp" n="SU-TI">Sorbonne Université - Texte Intégral</idno>
            <idno type="stamp" n="ALLIANCE-SU"> Alliance Sorbonne Université</idno>
            <idno type="stamp" n="INRIA-ETATSUNIS">Copublications Inria-Etats-Unis</idno>
            <idno type="stamp" n="SUPRA_MATHS_INFO">Mathématiques + Informatique</idno>
          </seriesStmt>
          <notesStmt/>
          <sourceDesc>
            <biblStruct>
              <analytic>
                <title xml:lang="en">Homogenized Transformers</title>
                <author role="aut">
                  <persName>
                    <forename type="first">Hugo</forename>
                    <surname>Koubbi</surname>
                  </persName>
                  <idno type="halauthorid">3260600-0</idno>
                  <affiliation ref="#struct-60"/>
                  <affiliation ref="#struct-1303693"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Borjan</forename>
                    <surname>Geshkovski</surname>
                  </persName>
                  <email type="md5">fc9f4eed97b14fe1624461ea234bab4f</email>
                  <email type="domain">inria.fr</email>
                  <idno type="idhal" notation="string">borjan-geshkovski</idno>
                  <idno type="idhal" notation="numeric">1053249</idno>
                  <idno type="halauthorid" notation="string">1384931-1053249</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-7890-3352</idno>
                  <affiliation ref="#struct-1005052"/>
                  <affiliation ref="#struct-1303693"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Philippe</forename>
                    <surname>Rigollet</surname>
                  </persName>
                  <email type="md5">7f986290561726ed1e7864f93b4adb07</email>
                  <email type="domain">mit.edu</email>
                  <idno type="idhal" notation="numeric">1665765</idno>
                  <idno type="halauthorid" notation="string">118324-1665765</idno>
                  <affiliation ref="#struct-103056"/>
                </author>
              </analytic>
              <monogr>
                <imprint>
                  <date type="datePub">2026-04-02</date>
                </imprint>
              </monogr>
              <idno type="arxiv">2604.01978</idno>
            </biblStruct>
          </sourceDesc>
          <profileDesc>
            <langUsage>
              <language ident="en">English</language>
            </langUsage>
            <textClass>
              <classCode scheme="halDomain" n="math">Mathematics [math]</classCode>
              <classCode scheme="halTypology" n="UNDEFINED">Preprints, Working Papers, ...</classCode>
              <classCode scheme="halOldTypology" n="UNDEFINED">Preprints, Working Papers, ...</classCode>
              <classCode scheme="halTreeTypology" n="UNDEFINED">Preprints, Working Papers, ...</classCode>
            </textClass>
            <abstract xml:lang="en">
              <p>We study a random model of deep multi-head self-attention in which the weights are resampled independently across layers and heads, as at initialization of training. Viewing depth as a time variable, the residual stream defines a discrete-time interacting particle system on the unit sphere. We prove that, under suitable joint scalings of the depth, the residual step size, and the number of heads, this dynamics admits a nontrivial homogenized limit. Depending on the scaling, the limit is either deterministic or stochastic with common noise; in the mean-field regime, the latter leads to a stochastic nonlinear Fokker--Planck equation for the conditional law of a representative token. In the Gaussian setting, the limiting drift vanishes, making the homogenized dynamics explicit enough to study representation collapse. This yields quantitative trade-offs between dimension, context length, and temperature, and identifies regimes in which clustering can be mitigated.</p>
            </abstract>
          </profileDesc>
        </biblFull>
      </listBibl>
    </body>
    <back>
      <listOrg type="structures">
        <org type="laboratory" xml:id="struct-60" status="VALID">
          <idno type="IdRef">088064239</idno>
          <idno type="ISNI">0000 0004 0645 4046</idno>
          <idno type="RNSR">199712592E</idno>
          <idno type="ROR">https://ror.org/03s0gj002</idno>
          <idno type="Wikidata">Q2913547</idno>
          <orgName>CEntre de REcherches en MAthématiques de la DEcision</orgName>
          <orgName type="acronym">CEREMADE</orgName>
          <date type="start">1971-01-01</date>
          <desc>
            <address>
              <addrLine>Place du Maréchal de Lattre de Tassigny 75775 - Paris Cedex 16</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.ceremade.dauphine.fr/index.html</ref>
          </desc>
          <listRelation>
            <relation active="#struct-300302" type="direct"/>
            <relation active="#struct-564132" type="indirect"/>
            <relation name="UMR7534 / URA749" active="#struct-441569" type="direct"/>
          </listRelation>
        </org>
        <org type="researchteam" xml:id="struct-1303693" status="VALID">
          <idno type="RNSR">202524704Y</idno>
          <orgName>Apprentissage Automatique et équations d’évolution</orgName>
          <orgName type="acronym">MEGAVOLT</orgName>
          <date type="start">2023-03-01</date>
          <date type="end">2029-05-31</date>
          <desc>
            <address>
              <addrLine>LJLL 4 Place Jussieu, 75005 Paris</addrLine>
              <country key="FR"/>
            </address>
          </desc>
          <listRelation>
            <relation active="#struct-1004954" type="direct"/>
            <relation name="UMR_8001" active="#struct-413221" type="indirect"/>
            <relation name="UMR8001" active="#struct-441569" type="indirect"/>
            <relation name="UMR_8001" active="#struct-557826" type="indirect"/>
            <relation active="#struct-1005052" type="direct"/>
            <relation active="#struct-413221" type="indirect"/>
            <relation name="UMR7598" active="#struct-441569" type="indirect"/>
            <relation name="UMR_7598" active="#struct-557826" type="indirect"/>
            <relation active="#struct-1175218" type="direct"/>
            <relation active="#struct-454310" type="indirect"/>
            <relation active="#struct-300009" type="indirect"/>
          </listRelation>
        </org>
        <org type="laboratory" xml:id="struct-1005052" status="VALID">
          <idno type="IdRef">099222221</idno>
          <idno type="ISNI">0000000101748385</idno>
          <idno type="RNSR">199712644L</idno>
          <idno type="ROR">https://ror.org/04xmteb38</idno>
          <orgName>Laboratoire Jacques-Louis Lions</orgName>
          <orgName type="acronym">LJLL (UMR_7598)</orgName>
          <date type="start">2020-01-01</date>
          <desc>
            <address>
              <addrLine>Sorbonne-Université, Boîte courrier 187 - 75252 Paris Cedex 05</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.ljll.math.upmc.fr</ref>
          </desc>
          <listRelation>
            <relation active="#struct-413221" type="direct"/>
            <relation name="UMR7598" active="#struct-441569" type="direct"/>
            <relation name="UMR_7598" active="#struct-557826" type="direct"/>
          </listRelation>
        </org>
        <org type="laboratory" xml:id="struct-103056" status="VALID">
          <orgName>Department of Mathematics [MIT]</orgName>
          <desc>
            <address>
              <addrLine>Headquarters Office Building 2, Room 236 77 Massachusetts Avenue Cambridge, MA 02139-4307</addrLine>
              <country key="US"/>
            </address>
            <ref type="url">http://math.mit.edu/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-301950" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-300302" status="VALID">
          <idno type="IdRef">027787109</idno>
          <idno type="ISNI">0000000120977052</idno>
          <idno type="ROR">https://ror.org/052bz7812</idno>
          <idno type="Wikidata">Q1546437</idno>
          <orgName>Université Paris Dauphine-PSL</orgName>
          <desc>
            <address>
              <addrLine>Place du Maréchal de Lattre de Tassigny75775 PARIS Cedex 16</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://dauphine.psl.eu/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-564132" type="direct"/>
          </listRelation>
        </org>
        <org type="regroupinstitution" xml:id="struct-564132" status="VALID">
          <idno type="IdRef">241597595</idno>
          <idno type="ISNI">0000 0004 1784 3645</idno>
          <idno type="ROR">https://ror.org/013cjyk83</idno>
          <orgName>Université Paris Sciences et Lettres</orgName>
          <orgName type="acronym">PSL</orgName>
          <desc>
            <address>
              <addrLine>60 rue Mazarine 75006 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.psl.eu/</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-441569" status="VALID">
          <idno type="IdRef">02636817X</idno>
          <idno type="ISNI">0000000122597504</idno>
          <idno type="ROR">https://ror.org/02feahw73</idno>
          <orgName>Centre National de la Recherche Scientifique</orgName>
          <orgName type="acronym">CNRS</orgName>
          <date type="start">1939-10-19</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.cnrs.fr/</ref>
          </desc>
        </org>
        <org type="laboratory" xml:id="struct-1004954" status="VALID">
          <idno type="IdRef">235620025</idno>
          <idno type="RNSR">201822759P</idno>
          <idno type="ROR">https://ror.org/02vnd0e65</idno>
          <orgName>Laboratoire de Probabilités, Statistique et Modélisation</orgName>
          <orgName type="acronym">LPSM (UMR_8001)</orgName>
          <date type="start">2020-01-01</date>
          <desc>
            <address>
              <addrLine>Campus Jussieu Tour 16-26, 1er étage 4, Place Jussieu 75005 Paris / Bâtiment Sophie Germain 5ème étage Avenue de France 75013 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.lpsm.paris</ref>
          </desc>
          <listRelation>
            <relation name="UMR_8001" active="#struct-413221" type="direct"/>
            <relation name="UMR8001" active="#struct-441569" type="direct"/>
            <relation name="UMR_8001" active="#struct-557826" type="direct"/>
          </listRelation>
        </org>
        <org type="regroupinstitution" xml:id="struct-413221" status="VALID">
          <idno type="IdRef">221333754</idno>
          <idno type="ROR">https://ror.org/02en5vm52</idno>
          <orgName>Sorbonne Université</orgName>
          <orgName type="acronym">SU</orgName>
          <date type="start">2018-01-01</date>
          <desc>
            <address>
              <addrLine>21 rue de l’École de médecine - 75006 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.sorbonne-universite.fr/</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-557826" status="VALID">
          <idno type="IdRef">236453505</idno>
          <idno type="ISNI">0000 0004 7885 7602</idno>
          <idno type="ROR">https://ror.org/05f82e368</idno>
          <orgName>Université Paris Cité</orgName>
          <orgName type="acronym">UPCité</orgName>
          <date type="start">2020-01-01</date>
          <desc>
            <address>
              <addrLine>85 boulevard Saint-Germain75006 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://u-paris.fr/</ref>
          </desc>
        </org>
        <org type="department" xml:id="struct-1175218" status="VALID">
          <idno type="ROR">https://ror.org/03cjm2544</idno>
          <orgName>Centre Inria de Sorbonne Université</orgName>
          <date type="start">2022-11-01</date>
          <desc>
            <address>
              <addrLine>48 Rue Barrault, 75013 Paris</addrLine>
              <country key="FR"/>
            </address>
          </desc>
          <listRelation>
            <relation active="#struct-454310" type="direct"/>
            <relation active="#struct-300009" type="indirect"/>
          </listRelation>
        </org>
        <org type="laboratory" xml:id="struct-454310" status="VALID">
          <idno type="IdRef">241614864</idno>
          <idno type="RNSR">196718247G</idno>
          <idno type="ROR">https://ror.org/05eyd5d35</idno>
          <orgName>Centre Inria de Paris</orgName>
          <date type="start">2016-03-10</date>
          <desc>
            <address>
              <addrLine>48 Rue Barrault, 75013 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/centre/paris</ref>
          </desc>
          <listRelation>
            <relation active="#struct-300009" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-300009" status="VALID">
          <idno type="ROR">https://ror.org/02kvxyf05</idno>
          <orgName>Institut National de Recherche en Informatique et en Automatique</orgName>
          <orgName type="acronym">Inria</orgName>
          <desc>
            <address>
              <addrLine>Domaine de VoluceauRocquencourt - BP 10578153 Le Chesnay Cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/en/</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-301950" status="VALID">
          <idno type="ROR">https://ror.org/042nb2s44</idno>
          <orgName>Massachusetts Institute of Technology</orgName>
          <orgName type="acronym">MIT</orgName>
          <desc>
            <address>
              <addrLine>77 Massachusetts Ave, Cambridge, MA 02139</addrLine>
              <country key="US"/>
            </address>
            <ref type="url">http://web.mit.edu</ref>
          </desc>
        </org>
      </listOrg>
    </back>
  </text>
</TEI>