<?xml version="1.0" encoding="utf-8"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hal="http://hal.archives-ouvertes.fr/" xmlns:gml="http://www.opengis.net/gml/3.3/" xmlns:gmlce="http://www.opengis.net/gml/3.3/ce" version="1.1" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 http://api.archives-ouvertes.fr/documents/aofr-sword.xsd">
  <teiHeader>
    <fileDesc>
      <titleStmt>
        <title>HAL TEI export of hal-04912112</title>
      </titleStmt>
      <publicationStmt>
        <distributor>CCSD</distributor>
        <availability status="restricted">
          <licence target="https://creativecommons.org/publicdomain/zero/1.0/">CC0 1.0 - Universal</licence>
        </availability>
        <date when="2026-05-03T12:13:09+02:00"/>
      </publicationStmt>
      <sourceDesc>
        <p part="N">HAL API Platform</p>
      </sourceDesc>
    </fileDesc>
  </teiHeader>
  <text>
    <body>
      <listBibl>
        <biblFull>
          <titleStmt>
            <title xml:lang="en">Probing Whisper Predictions for French, English and Persian Transcriptions</title>
            <author role="aut">
              <persName>
                <forename type="first">Nicolas</forename>
                <surname>Ballier</surname>
              </persName>
              <email type="md5">86de69c2eafb1aa83f9fa105775d7ac0</email>
              <email type="domain">univ-paris-diderot.fr</email>
              <idno type="idhal" notation="string">nicolas-ballier</idno>
              <idno type="idhal" notation="numeric">7391</idno>
              <idno type="halauthorid" notation="string">22805-7391</idno>
              <idno type="ORCID">https://orcid.org/0000-0003-2179-1043</idno>
              <idno type="GOOGLE SCHOLAR">https://scholar.google.fr/citations?user=1DqF0BgAAAAJ&amp;hl=fr</idno>
              <idno type="IDREF">https://www.idref.fr/057712409</idno>
              <idno type="GOOGLE SCHOLAR">https://scholar.google.fr/citations?user=1DqF0BgAAAAJ</idno>
              <affiliation ref="#struct-1005004"/>
              <affiliation ref="#struct-1004855"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Léa</forename>
                <surname>Burin</surname>
              </persName>
              <idno type="halauthorid">1161267-0</idno>
              <affiliation ref="#struct-1005004"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Behnoosh</forename>
                <surname>Namdarzadeh</surname>
              </persName>
              <email type="md5">03b6cfb7fa41379dbb77ef5ab7ebe5bd</email>
              <email type="domain">etu.u-paris.fr</email>
              <idno type="idhal" notation="string">behnoosh-namdarzadeh</idno>
              <idno type="idhal" notation="numeric">1209570</idno>
              <idno type="halauthorid" notation="string">2680465-1209570</idno>
              <idno type="ORCID">https://orcid.org/0009-0000-8730-7766</idno>
              <affiliation ref="#struct-1067370"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Sara</forename>
                <surname>Ng</surname>
              </persName>
              <idno type="halauthorid">3388331-0</idno>
              <affiliation ref="#struct-300433"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Richard</forename>
                <surname>Wright</surname>
              </persName>
              <idno type="halauthorid">703075-0</idno>
              <affiliation ref="#struct-300433"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Jean-Baptiste</forename>
                <surname>Yunès</surname>
              </persName>
              <email type="md5">5733fc0a14ed6182e2a30b1f3e2b94c7</email>
              <email type="domain">u-paris.fr</email>
              <idno type="idhal" notation="string">jean-baptiste-yunes</idno>
              <idno type="idhal" notation="numeric">4898</idno>
              <idno type="halauthorid" notation="string">14881-4898</idno>
              <idno type="IDREF">https://www.idref.fr/057435960</idno>
              <idno type="ORCID">https://orcid.org/0000-0003-0291-4263</idno>
              <affiliation ref="#struct-444497"/>
            </author>
            <editor role="depositor">
              <persName>
                <forename>Nicolas</forename>
                <surname>BALLIER</surname>
              </persName>
              <email type="md5">4ba02dedb968400f01261e0baafc107a</email>
              <email type="domain">u-paris.fr</email>
            </editor>
          </titleStmt>
          <editionStmt>
            <edition n="v1" type="current">
              <date type="whenSubmitted">2025-01-25 18:36:13</date>
              <date type="whenWritten">2024</date>
              <date type="whenModified">2025-01-31 03:25:10</date>
              <date type="whenReleased">2025-01-29 10:06:44</date>
              <date type="whenProduced">2024-10-19</date>
              <date type="whenEndEmbargoed">2025-01-25</date>
              <ref type="file" target="https://hal.science/hal-04912112v1/document">
                <date notBefore="2025-01-25"/>
              </ref>
              <ref type="file" subtype="author" n="1" target="https://hal.science/hal-04912112v1/file/Ballier_etal_2024ProbingWhisper.pdf" id="file-4912112-4269618">
                <date notBefore="2025-01-25"/>
              </ref>
            </edition>
            <respStmt>
              <resp>contributor</resp>
              <name key="306734">
                <persName>
                  <forename>Nicolas</forename>
                  <surname>BALLIER</surname>
                </persName>
                <email type="md5">4ba02dedb968400f01261e0baafc107a</email>
                <email type="domain">u-paris.fr</email>
              </name>
            </respStmt>
          </editionStmt>
          <publicationStmt>
            <distributor>CCSD</distributor>
            <idno type="halId">hal-04912112</idno>
            <idno type="halUri">https://hal.science/hal-04912112</idno>
            <idno type="halBibtex">ballier:hal-04912112</idno>
            <idno type="halRefHtml">&lt;i&gt;7the International Conference on Natural Language and Speech Processing&lt;/i&gt;, Mourad Abbas, Oct 2024, Trente, Italy. pp.129-138</idno>
            <idno type="halRef">7the International Conference on Natural Language and Speech Processing, Mourad Abbas, Oct 2024, Trente, Italy. pp.129-138</idno>
            <availability status="restricted">
              <licence target="https://creativecommons.org/licenses/by/4.0/">CC BY 4.0 - Attribution<ref corresp="#file-4912112-4269618"/></licence>
            </availability>
          </publicationStmt>
          <seriesStmt>
            <idno type="stamp" n="CNRS">CNRS - Centre national de la recherche scientifique</idno>
            <idno type="stamp" n="LLF">Laboratoire de Linguistique Formelle</idno>
            <idno type="stamp" n="CAMPUS-AAR">Campus-AAR</idno>
            <idno type="stamp" n="AAI">Collection LaCAS - Open Archives in Language and Cultural Area Studies</idno>
            <idno type="stamp" n="CLILLAC-ARP">Productions scientifiques du CLILLAC-ARP</idno>
            <idno type="stamp" n="UNIV-PARIS">Université Paris Cité</idno>
            <idno type="stamp" n="UNIVERSITE-PARIS" corresp="UNIV-PARIS">Université Paris Cité</idno>
            <idno type="stamp" n="UP-SCIENCES">Université Paris Cité - Faculté des Sciences</idno>
            <idno type="stamp" n="UP-SOCIETES-HUMANITES">Université Paris Cité - Faculté Sociétés et Humanités</idno>
            <idno type="stamp" n="IRIF">Institut de Recherche en Informatique Fondamentale</idno>
            <idno type="stamp" n="ALTAE">Approches Linguistiques Théoriques, Appliquées et Expérimentales : langues et cultures connectées</idno>
          </seriesStmt>
          <notesStmt>
            <note type="audience" n="2">International</note>
            <note type="invited" n="0">No</note>
            <note type="popular" n="0">No</note>
            <note type="peer" n="1">Yes</note>
            <note type="proceedings" n="1">Yes</note>
          </notesStmt>
          <sourceDesc>
            <biblStruct>
              <analytic>
                <title xml:lang="en">Probing Whisper Predictions for French, English and Persian Transcriptions</title>
                <author role="aut">
                  <persName>
                    <forename type="first">Nicolas</forename>
                    <surname>Ballier</surname>
                  </persName>
                  <email type="md5">86de69c2eafb1aa83f9fa105775d7ac0</email>
                  <email type="domain">univ-paris-diderot.fr</email>
                  <idno type="idhal" notation="string">nicolas-ballier</idno>
                  <idno type="idhal" notation="numeric">7391</idno>
                  <idno type="halauthorid" notation="string">22805-7391</idno>
                  <idno type="ORCID">https://orcid.org/0000-0003-2179-1043</idno>
                  <idno type="GOOGLE SCHOLAR">https://scholar.google.fr/citations?user=1DqF0BgAAAAJ&amp;hl=fr</idno>
                  <idno type="IDREF">https://www.idref.fr/057712409</idno>
                  <idno type="GOOGLE SCHOLAR">https://scholar.google.fr/citations?user=1DqF0BgAAAAJ</idno>
                  <affiliation ref="#struct-1005004"/>
                  <affiliation ref="#struct-1004855"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Léa</forename>
                    <surname>Burin</surname>
                  </persName>
                  <idno type="halauthorid">1161267-0</idno>
                  <affiliation ref="#struct-1005004"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Behnoosh</forename>
                    <surname>Namdarzadeh</surname>
                  </persName>
                  <email type="md5">03b6cfb7fa41379dbb77ef5ab7ebe5bd</email>
                  <email type="domain">etu.u-paris.fr</email>
                  <idno type="idhal" notation="string">behnoosh-namdarzadeh</idno>
                  <idno type="idhal" notation="numeric">1209570</idno>
                  <idno type="halauthorid" notation="string">2680465-1209570</idno>
                  <idno type="ORCID">https://orcid.org/0009-0000-8730-7766</idno>
                  <affiliation ref="#struct-1067370"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Sara</forename>
                    <surname>Ng</surname>
                  </persName>
                  <idno type="halauthorid">3388331-0</idno>
                  <affiliation ref="#struct-300433"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Richard</forename>
                    <surname>Wright</surname>
                  </persName>
                  <idno type="halauthorid">703075-0</idno>
                  <affiliation ref="#struct-300433"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Jean-Baptiste</forename>
                    <surname>Yunès</surname>
                  </persName>
                  <email type="md5">5733fc0a14ed6182e2a30b1f3e2b94c7</email>
                  <email type="domain">u-paris.fr</email>
                  <idno type="idhal" notation="string">jean-baptiste-yunes</idno>
                  <idno type="idhal" notation="numeric">4898</idno>
                  <idno type="halauthorid" notation="string">14881-4898</idno>
                  <idno type="IDREF">https://www.idref.fr/057435960</idno>
                  <idno type="ORCID">https://orcid.org/0000-0003-0291-4263</idno>
                  <affiliation ref="#struct-444497"/>
                </author>
              </analytic>
              <monogr>
                <title level="m">Proceedings of the 7th International Conference on Natural Language and Speech Processing (ICNLSP 2024)</title>
                <meeting>
                  <title>7the International Conference on Natural Language and Speech Processing</title>
                  <date type="start">2024-10-19</date>
                  <date type="end">2024-10-20</date>
                  <settlement>Trente</settlement>
                  <country key="IT">Italy</country>
                </meeting>
                <respStmt>
                  <resp>conferenceOrganizer</resp>
                  <name>Mourad Abbas</name>
                </respStmt>
                <editor>Mourad Abbas, Abed Alhakim Freihat</editor>
                <imprint>
                  <publisher>Association for Computational Linguistics</publisher>
                  <biblScope unit="serie">2024 International Conference on Natural Language and Speech Processing (ICNLSP 2024)</biblScope>
                  <biblScope unit="volume">7</biblScope>
                  <biblScope unit="pp">129-138</biblScope>
                  <date type="datePub">2024</date>
                </imprint>
              </monogr>
              <ref type="publisher">https://www.icnlsp.org/2024welcome/</ref>
            </biblStruct>
          </sourceDesc>
          <profileDesc>
            <langUsage>
              <language ident="en">English</language>
            </langUsage>
            <textClass>
              <keywords scheme="author">
                <term xml:lang="en">Probing</term>
                <term xml:lang="en">English</term>
                <term xml:lang="en">French</term>
                <term xml:lang="en">Persian</term>
                <term xml:lang="en">ASR Automatic Speech Recognition</term>
                <term xml:lang="en">Whisper</term>
                <term xml:lang="en">LLM calibration</term>
                <term xml:lang="fr">LLM audio</term>
                <term xml:lang="fr">persan</term>
                <term xml:lang="fr">français</term>
                <term xml:lang="fr">anglais</term>
                <term xml:lang="fr">sondes</term>
                <term xml:lang="fr">calibrage de LLM</term>
              </keywords>
              <classCode scheme="halDomain" n="info.info-cl">Computer Science [cs]/Computation and Language [cs.CL]</classCode>
              <classCode scheme="halTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halOldTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halTreeTypology" n="COMM">Conference papers</classCode>
            </textClass>
            <abstract xml:lang="en">
              <p>Whisper is a widely-used open-access Large Language Model (LLM) trained using a multilingual paradigm. As such it represents an important opportunity for researchers to study how multilingual LLMs function across languages. In this paper, we analyse Whisper's Large and Medium models for Persian, English and French using a transcription task. To investigate the calibration of Whisper models, we use a customised C++ version of Whisper to probe Whisper's internal representations by extracting the subtoken probabilities for transcriptions of speech samples of the target languages. We discuss our subtoken-based evaluation of prediction accuracy as a proxy for standard Word Error Rate evaluation of the different Whisper models. The accuracy of the ASR predictions is investigated as a function of target language and part of speech. Our analysis reveals an architectural bias for French and discrepancies in accuracy in relation to the size of the training data. The results of our novel subtoken-based evaluation supplement previously-reported cross-lingual evaluations of Whisper, and enable better fine-tuning by suggesting types of data that may improve calibration.</p>
            </abstract>
            <abstract xml:lang="fr">
              <p>Whisper est un grand modèle de langage (LLM) largement utilisé et en accès libre, entraîné à partir de plusieurs langues. En tant que tel, il donne l'occasion aux chercheurs d'étudier le fonctionnement des LLM multilingues à travers les langues. Dans cet article, nous analysons les modèles large et medium de Whisper pour le persan, l'anglais et le français à partir d'une tâche de transcription. Pour étudier le calibrage des modèles de Whisper, nous utilisons une version C++ de Whisper personnalisée pour sonder les représentations internes de Whisper, en extrayant les probabilités des sous-tokens pour les transcriptions d'échantillons de discours des langues cibles. Nous discutons de la précision des prédictions fondée sur les sous-tokens et la comparons avec l'évaluation standard (WER) des différents modèles de Whisper. Notre analyse révèle un biais architectural pour le français et nous montrons que des différentiels de précision sont en rapport avec la taille des données d'entraînement.</p>
            </abstract>
          </profileDesc>
        </biblFull>
      </listBibl>
    </body>
    <back>
      <listOrg type="structures">
        <org type="laboratory" xml:id="struct-1005004" status="OLD">
          <idno type="IdRef">12619601X</idno>
          <idno type="RNSR">200515225G</idno>
          <orgName>Centre de Linguistique Inter-langues, de Lexicologie, de Linguistique Anglaise et de Corpus</orgName>
          <orgName type="acronym">CLILLAC-ARP (URP_3967)</orgName>
          <date type="start">2020-01-01</date>
          <date type="end">2024-12-31</date>
          <desc>
            <address>
              <addrLine>Université Paris Cité, Bât. Olympe de Gouges, case postale 7046, 75205 Paris CEDEX13</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://clillac-arp.u-paris.fr/</ref>
          </desc>
          <listRelation>
            <relation name="URP_3967" active="#struct-557826" type="direct"/>
          </listRelation>
        </org>
        <org type="laboratory" xml:id="struct-1004855" status="VALID">
          <idno type="IdRef">026390574</idno>
          <idno type="RNSR">200112497J</idno>
          <idno type="ROR">https://ror.org/0223bz716</idno>
          <orgName>Laboratoire de Linguistique Formelle</orgName>
          <orgName type="acronym">LLF - UMR7110</orgName>
          <date type="start">2020-01-01</date>
          <desc>
            <address>
              <addrLine>8 Rue Albert Einstein, 75013 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.llf.cnrs.fr/</ref>
          </desc>
          <listRelation>
            <relation name="UMR7110" active="#struct-441569" type="direct"/>
            <relation name="UMR_7110" active="#struct-557826" type="direct"/>
          </listRelation>
        </org>
        <org type="regrouplaboratory" xml:id="struct-1067370" status="VALID">
          <orgName>UFR Linguistique [Sociétés et Humanités] - Université Paris Cité</orgName>
          <orgName type="acronym">UFR Linguistique UPCité</orgName>
          <date type="start">2020-01-01</date>
          <desc>
            <address>
              <addrLine>Bâtiment Olympe de Gouges – 8 place Paul-Ricœur, 75013 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://u-paris.fr/linguistique/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-557826" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-300433" status="VALID">
          <idno type="ROR">https://ror.org/00cvxb145</idno>
          <orgName>University of Washington [Seattle]</orgName>
          <desc>
            <address>
              <addrLine>Seattle, Washington 98105</addrLine>
              <country key="US"/>
            </address>
            <ref type="url">http://www.washington.edu/</ref>
          </desc>
        </org>
        <org type="laboratory" xml:id="struct-444497" status="OLD">
          <idno type="IdRef">193993007</idno>
          <idno type="RNSR">201621976X</idno>
          <orgName>Institut de Recherche en Informatique Fondamentale</orgName>
          <orgName type="acronym">IRIF (UMR_8243)</orgName>
          <date type="start">2016-01-01</date>
          <date type="end">2019-12-31</date>
          <desc>
            <address>
              <addrLine>Université de ParisBâtiment Sophie Germain, Case courrier 70148 Place Aurélie Nemours75205 Paris Cedex 13</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.irif.fr/</ref>
          </desc>
          <listRelation>
            <relation name="UMR_8243" active="#struct-300301" type="direct"/>
            <relation name="UMR8243" active="#struct-441569" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-557826" status="VALID">
          <idno type="IdRef">236453505</idno>
          <idno type="ISNI">0000 0004 7885 7602</idno>
          <idno type="ROR">https://ror.org/05f82e368</idno>
          <orgName>Université Paris Cité</orgName>
          <orgName type="acronym">UPCité</orgName>
          <date type="start">2020-01-01</date>
          <desc>
            <address>
              <addrLine>85 boulevard Saint-Germain75006 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://u-paris.fr/</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-441569" status="VALID">
          <idno type="IdRef">02636817X</idno>
          <idno type="ISNI">0000000122597504</idno>
          <idno type="ROR">https://ror.org/02feahw73</idno>
          <orgName>Centre National de la Recherche Scientifique</orgName>
          <orgName type="acronym">CNRS</orgName>
          <date type="start">1939-10-19</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.cnrs.fr/</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-300301" status="OLD">
          <idno type="IdRef">027542084</idno>
          <idno type="ISNI">0000000121514068</idno>
          <idno type="ROR">https://ror.org/02n7qrg46</idno>
          <orgName>Université Paris Diderot - Paris 7</orgName>
          <orgName type="acronym">UPD7</orgName>
          <date type="end">2019-12-31</date>
          <desc>
            <address>
              <addrLine>5 rue Thomas-Mann - 75205 Paris cedex 13</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.univ-paris-diderot.fr</ref>
          </desc>
        </org>
      </listOrg>
    </back>
  </text>
</TEI>