<?xml version="1.0" encoding="utf-8"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hal="http://hal.archives-ouvertes.fr/" xmlns:gml="http://www.opengis.net/gml/3.3/" xmlns:gmlce="http://www.opengis.net/gml/3.3/ce" version="1.1" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 http://api.archives-ouvertes.fr/documents/aofr-sword.xsd">
  <teiHeader>
    <fileDesc>
      <titleStmt>
        <title>HAL TEI export of hal-05365631</title>
      </titleStmt>
      <publicationStmt>
        <distributor>CCSD</distributor>
        <availability status="restricted">
          <licence target="https://creativecommons.org/publicdomain/zero/1.0/">CC0 1.0 - Universal</licence>
        </availability>
        <date when="2026-05-26T05:26:18+02:00"/>
      </publicationStmt>
      <sourceDesc>
        <p part="N">HAL API Platform</p>
      </sourceDesc>
    </fileDesc>
  </teiHeader>
  <text>
    <body>
      <listBibl>
        <biblFull>
          <titleStmt>
            <title xml:lang="en">ModernBERT or DeBERTaV3? Examining Architecture and Data Influence on Transformer Encoder Models Performance</title>
            <author role="aut">
              <persName>
                <forename type="first">Wissam</forename>
                <surname>Antoun</surname>
              </persName>
              <email type="md5">63950a3b5625439fa32a52b29ff4f0b2</email>
              <email type="domain">inria.fr</email>
              <idno type="idhal" notation="string">wissam-antoun</idno>
              <idno type="idhal" notation="numeric">1214021</idno>
              <idno type="halauthorid" notation="string">2691933-1214021</idno>
              <idno type="ARXIV">https://arxiv.org/a/antoun_w_1</idno>
              <idno type="GOOGLE SCHOLAR">fEgsyU4AAAAJ</idno>
              <idno type="ORCID">https://orcid.org/0000-0001-8021-5834</idno>
              <affiliation ref="#struct-482775"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Benoît</forename>
                <surname>Sagot</surname>
              </persName>
              <email type="md5">ef534712ca682bf74ec7eef17d3d1b5f</email>
              <email type="domain">inria.fr</email>
              <idno type="idhal" notation="string">bsagot</idno>
              <idno type="idhal" notation="numeric">1461</idno>
              <idno type="halauthorid" notation="string">17075-1461</idno>
              <idno type="IDREF">https://www.idref.fr/177454229</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-0107-8526</idno>
              <idno type="ARXIV">https://arxiv.org/a/sagot_b_1</idno>
              <idno type="GOOGLE SCHOLAR">HXUT9ZkAAAAJ</idno>
              <affiliation ref="#struct-482775"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Djamé</forename>
                <surname>Seddah</surname>
              </persName>
              <email type="md5">1a46dccff44eefe8ac69598e3b186e83</email>
              <email type="domain">paris-sorbonne.fr</email>
              <idno type="idhal" notation="string">djameseddah</idno>
              <idno type="idhal" notation="numeric">11545</idno>
              <idno type="halauthorid" notation="string">1878-11545</idno>
              <idno type="IDREF">https://www.idref.fr/086185136</idno>
              <idno type="ORCID">https://orcid.org/0000-0003-0194-8197</idno>
              <affiliation ref="#struct-482775"/>
            </author>
            <editor role="depositor">
              <persName>
                <forename>Wissam</forename>
                <surname>Antoun</surname>
              </persName>
              <email type="md5">63950a3b5625439fa32a52b29ff4f0b2</email>
              <email type="domain">inria.fr</email>
            </editor>
            <funder ref="#projanr-185715"/>
            <funder>projet BPI "Scribe"</funder>
          </titleStmt>
          <editionStmt>
            <edition n="v1" type="current">
              <date type="whenSubmitted">2025-11-14 14:52:24</date>
              <date type="whenModified">2026-02-07 05:32:55</date>
              <date type="whenReleased">2025-11-14 16:47:54</date>
              <date type="whenProduced">2025-12-20</date>
              <date type="whenEndEmbargoed">2025-11-14</date>
              <ref type="file" target="https://hal.science/hal-05365631v1/document">
                <date notBefore="2025-11-14"/>
              </ref>
              <ref type="file" subtype="author" n="1" target="https://hal.science/hal-05365631v1/file/jkdkxjhgmmpjxkygwbgcscctxzzzkwqf.pdf" id="file-5365631-4590408">
                <date notBefore="2025-11-14"/>
              </ref>
            </edition>
            <respStmt>
              <resp>contributor</resp>
              <name key="1372417">
                <persName>
                  <forename>Wissam</forename>
                  <surname>Antoun</surname>
                </persName>
                <email type="md5">63950a3b5625439fa32a52b29ff4f0b2</email>
                <email type="domain">inria.fr</email>
              </name>
            </respStmt>
          </editionStmt>
          <publicationStmt>
            <distributor>CCSD</distributor>
            <idno type="halId">hal-05365631</idno>
            <idno type="halUri">https://hal.science/hal-05365631</idno>
            <idno type="halBibtex">antoun:hal-05365631</idno>
            <idno type="halRefHtml">&lt;i&gt;IJCNLP-AACL 2025 - 14th International Joint Conference on Natural Language Processing (IJCNLP) and The 4th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (AACL)&lt;/i&gt;, Dec 2025, Mumbai, India</idno>
            <idno type="halRef">IJCNLP-AACL 2025 - 14th International Joint Conference on Natural Language Processing (IJCNLP) and The 4th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (AACL), Dec 2025, Mumbai, India</idno>
            <availability status="restricted">
              <licence target="https://creativecommons.org/licenses/by/4.0/">CC BY 4.0 - Attribution<ref corresp="#file-5365631-4590408"/></licence>
            </availability>
          </publicationStmt>
          <seriesStmt>
            <idno type="stamp" n="INRIA">INRIA - Institut National de Recherche en Informatique et en Automatique</idno>
            <idno type="stamp" n="INRIA-ROCQ">INRIA Paris - Rocquencourt</idno>
            <idno type="stamp" n="TESTALAIN1">TESTALAIN1</idno>
            <idno type="stamp" n="INRIA2">INRIA 2</idno>
            <idno type="stamp" n="ANR">ANR</idno>
            <idno type="stamp" n="ANR-IA-23" corresp="ANR-IA">ANR-IA-23</idno>
            <idno type="stamp" n="ANR-IA">ANR-IA</idno>
          </seriesStmt>
          <notesStmt>
            <note type="audience" n="2">International</note>
            <note type="invited" n="0">No</note>
            <note type="popular" n="0">No</note>
            <note type="peer" n="1">Yes</note>
            <note type="proceedings" n="1">Yes</note>
          </notesStmt>
          <sourceDesc>
            <biblStruct>
              <analytic>
                <title xml:lang="en">ModernBERT or DeBERTaV3? Examining Architecture and Data Influence on Transformer Encoder Models Performance</title>
                <author role="aut">
                  <persName>
                    <forename type="first">Wissam</forename>
                    <surname>Antoun</surname>
                  </persName>
                  <email type="md5">63950a3b5625439fa32a52b29ff4f0b2</email>
                  <email type="domain">inria.fr</email>
                  <idno type="idhal" notation="string">wissam-antoun</idno>
                  <idno type="idhal" notation="numeric">1214021</idno>
                  <idno type="halauthorid" notation="string">2691933-1214021</idno>
                  <idno type="ARXIV">https://arxiv.org/a/antoun_w_1</idno>
                  <idno type="GOOGLE SCHOLAR">fEgsyU4AAAAJ</idno>
                  <idno type="ORCID">https://orcid.org/0000-0001-8021-5834</idno>
                  <affiliation ref="#struct-482775"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Benoît</forename>
                    <surname>Sagot</surname>
                  </persName>
                  <email type="md5">ef534712ca682bf74ec7eef17d3d1b5f</email>
                  <email type="domain">inria.fr</email>
                  <idno type="idhal" notation="string">bsagot</idno>
                  <idno type="idhal" notation="numeric">1461</idno>
                  <idno type="halauthorid" notation="string">17075-1461</idno>
                  <idno type="IDREF">https://www.idref.fr/177454229</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-0107-8526</idno>
                  <idno type="ARXIV">https://arxiv.org/a/sagot_b_1</idno>
                  <idno type="GOOGLE SCHOLAR">HXUT9ZkAAAAJ</idno>
                  <affiliation ref="#struct-482775"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Djamé</forename>
                    <surname>Seddah</surname>
                  </persName>
                  <email type="md5">1a46dccff44eefe8ac69598e3b186e83</email>
                  <email type="domain">paris-sorbonne.fr</email>
                  <idno type="idhal" notation="string">djameseddah</idno>
                  <idno type="idhal" notation="numeric">11545</idno>
                  <idno type="halauthorid" notation="string">1878-11545</idno>
                  <idno type="IDREF">https://www.idref.fr/086185136</idno>
                  <idno type="ORCID">https://orcid.org/0000-0003-0194-8197</idno>
                  <affiliation ref="#struct-482775"/>
                </author>
              </analytic>
              <monogr>
                <meeting>
                  <title>IJCNLP-AACL 2025 - 14th International Joint Conference on Natural Language Processing (IJCNLP) and The 4th Conference of the Asia-Pacific Chapter of the Association for Computational Linguistics (AACL)</title>
                  <date type="start">2025-12-20</date>
                  <date type="end">2025-11-24</date>
                  <settlement>Mumbai</settlement>
                  <country key="IN">India</country>
                </meeting>
                <imprint>
                  <date type="datePub">2025-11-14</date>
                </imprint>
              </monogr>
            </biblStruct>
          </sourceDesc>
          <profileDesc>
            <langUsage>
              <language ident="en">English</language>
            </langUsage>
            <textClass>
              <keywords scheme="author">
                <term xml:lang="en">camembert</term>
                <term xml:lang="en">nlp</term>
                <term xml:lang="en">french</term>
                <term xml:lang="en">deberta</term>
                <term xml:lang="en">bert</term>
                <term xml:lang="en">modernbert</term>
              </keywords>
              <classCode scheme="halDomain" n="info.info-cl">Computer Science [cs]/Computation and Language [cs.CL]</classCode>
              <classCode scheme="halTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halOldTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halTreeTypology" n="COMM">Conference papers</classCode>
            </textClass>
            <abstract xml:lang="en">
              <p>Pretrained transformer-encoder models like DeBERTaV3 and ModernBERT introduce architectural advancements aimed at improving efficiency and performance. Although the authors of ModernBERT report improved performance over DeBERTaV3 on several benchmarks, the lack of disclosed training data and the absence of comparisons using a shared dataset make it difficult to determine whether these gains are due to architectural improvements or differences in training data. In this work, we conduct a controlled study by pretraining ModernBERT on the same dataset as CamemBERTaV2, a DeBERTaV3 French model, isolating the effect of model design.&lt;p&gt;Our results show that the previous model generation remains superior in sample efficiency and overall benchmark performance, with ModernBERT's primary advantage being its support for long context, faster training, and inference speed. However, the new proposed model still provides meaningful architectural improvements compared to earlier models such as BERT and RoBERTa. Additionally, we observe that high-quality pre-training data accelerates convergence but does not significantly improve final performance, suggesting potential benchmark saturation. These findings show the importance of disentangling pretraining data from architectural innovations when evaluating transformer models.&lt;/p&gt;</p>
            </abstract>
          </profileDesc>
        </biblFull>
      </listBibl>
    </body>
    <back>
      <listOrg type="structures">
        <org type="researchteam" xml:id="struct-482775" status="VALID">
          <idno type="RNSR">201722248N</idno>
          <idno type="ROR">https://ror.org/05rnwy495</idno>
          <orgName>Automatic Language Modelling and ANAlysis &amp; Computational Humanities</orgName>
          <orgName type="acronym">ALMAnaCH</orgName>
          <date type="start">2017-01-01</date>
          <date type="end">2029-04-30</date>
          <desc>
            <address>
              <addrLine>48 Rue Barrault, 75013 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.inria.fr/equipes/almanach</ref>
          </desc>
          <listRelation>
            <relation active="#struct-454310" type="direct"/>
            <relation active="#struct-300009" type="indirect"/>
          </listRelation>
        </org>
        <org type="laboratory" xml:id="struct-454310" status="VALID">
          <idno type="IdRef">241614864</idno>
          <idno type="RNSR">196718247G</idno>
          <idno type="ROR">https://ror.org/05eyd5d35</idno>
          <orgName>Centre Inria de Paris</orgName>
          <date type="start">2016-03-10</date>
          <desc>
            <address>
              <addrLine>48 Rue Barrault, 75013 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/centre/paris</ref>
          </desc>
          <listRelation>
            <relation active="#struct-300009" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-300009" status="VALID">
          <idno type="ROR">https://ror.org/02kvxyf05</idno>
          <orgName>Institut National de Recherche en Informatique et en Automatique</orgName>
          <orgName type="acronym">Inria</orgName>
          <desc>
            <address>
              <addrLine>Domaine de VoluceauRocquencourt - BP 10578153 Le Chesnay Cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/en/</ref>
          </desc>
        </org>
      </listOrg>
      <listOrg type="projects">
        <org type="anrProject" xml:id="projanr-185715" status="VALID">
          <idno type="anr">ANR-23-IACL-0008</idno>
          <orgName>PR[AI]RIE-PSAI</orgName>
          <desc>PR[AI]RIE-PSAI - Paris School of Artificial Intelligence</desc>
          <date type="start">2023</date>
        </org>
      </listOrg>
    </back>
  </text>
</TEI>