<?xml version="1.0" encoding="utf-8"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hal="http://hal.archives-ouvertes.fr/" xmlns:gml="http://www.opengis.net/gml/3.3/" xmlns:gmlce="http://www.opengis.net/gml/3.3/ce" version="1.1" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 http://api.archives-ouvertes.fr/documents/aofr-sword.xsd">
  <teiHeader>
    <fileDesc>
      <titleStmt>
        <title>HAL TEI export of hal-04300824</title>
      </titleStmt>
      <publicationStmt>
        <distributor>CCSD</distributor>
        <availability status="restricted">
          <licence target="https://creativecommons.org/publicdomain/zero/1.0/">CC0 1.0 - Universal</licence>
        </availability>
        <date when="2026-05-03T14:06:16+02:00"/>
      </publicationStmt>
      <sourceDesc>
        <p part="N">HAL API Platform</p>
      </sourceDesc>
    </fileDesc>
  </teiHeader>
  <text>
    <body>
      <listBibl>
        <biblFull>
          <titleStmt>
            <title xml:lang="en">RoCS-MT: Robustness Challenge Set for Machine Translation</title>
            <author role="aut">
              <persName>
                <forename type="first">Rachel</forename>
                <surname>Bawden</surname>
              </persName>
              <email type="md5">a08b8a5ca6961e673dd659fdfe33c881</email>
              <email type="domain">inria.fr</email>
              <idno type="idhal" notation="string">rachel-bawden</idno>
              <idno type="idhal" notation="numeric">9441</idno>
              <idno type="halauthorid" notation="string">28540-9441</idno>
              <idno type="ORCID">https://orcid.org/0000-0001-9553-1768</idno>
              <idno type="IDREF">https://www.idref.fr/233174591</idno>
              <idno type="ARXIV">https://arxiv.org/a/bawden_r_1</idno>
              <affiliation ref="#struct-482775"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Benoît</forename>
                <surname>Sagot</surname>
              </persName>
              <email type="md5">ef534712ca682bf74ec7eef17d3d1b5f</email>
              <email type="domain">inria.fr</email>
              <idno type="idhal" notation="string">bsagot</idno>
              <idno type="idhal" notation="numeric">1461</idno>
              <idno type="halauthorid" notation="string">17075-1461</idno>
              <idno type="IDREF">https://www.idref.fr/177454229</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-0107-8526</idno>
              <idno type="ARXIV">https://arxiv.org/a/sagot_b_1</idno>
              <idno type="GOOGLE SCHOLAR">HXUT9ZkAAAAJ</idno>
              <affiliation ref="#struct-482775"/>
            </author>
            <editor role="depositor">
              <persName>
                <forename>Rachel</forename>
                <surname>Bawden</surname>
              </persName>
              <email type="md5">a08b8a5ca6961e673dd659fdfe33c881</email>
              <email type="domain">inria.fr</email>
            </editor>
            <funder ref="#projanr-50388"/>
            <funder>This paper was funded by both authors chair positions in the PRAIRIE institute funded by the French national agency ANR as part of the “Investissements d’avenir” programme under the reference ANR-19-P3IA-0001. It was also funded by Rachel Bawden’s Emergence project, DadaNMT, funded by Sorbonne Université.</funder>
          </titleStmt>
          <editionStmt>
            <edition n="v1" type="current">
              <date type="whenSubmitted">2023-11-22 17:06:46</date>
              <date type="whenModified">2025-09-12 17:04:02</date>
              <date type="whenReleased">2023-11-23 10:38:10</date>
              <date type="whenProduced">2023-12-06</date>
              <date type="whenEndEmbargoed">2023-11-22</date>
              <ref type="file" target="https://hal.science/hal-04300824v1/document">
                <date notBefore="2023-11-22"/>
              </ref>
              <ref type="file" subtype="author" n="1" target="https://hal.science/hal-04300824v1/file/Article_RoCS_MT-7.pdf" id="file-4300824-3751844">
                <date notBefore="2023-11-22"/>
              </ref>
            </edition>
            <respStmt>
              <resp>contributor</resp>
              <name key="197025">
                <persName>
                  <forename>Rachel</forename>
                  <surname>Bawden</surname>
                </persName>
                <email type="md5">a08b8a5ca6961e673dd659fdfe33c881</email>
                <email type="domain">inria.fr</email>
              </name>
            </respStmt>
          </editionStmt>
          <publicationStmt>
            <distributor>CCSD</distributor>
            <idno type="halId">hal-04300824</idno>
            <idno type="halUri">https://hal.science/hal-04300824</idno>
            <idno type="halBibtex">bawden:hal-04300824</idno>
            <idno type="halRefHtml">&lt;i&gt;WMT23 - Eighth Conference on Machine Translation&lt;/i&gt;, Dec 2023, Singapore, Singapore. pp.198--216</idno>
            <idno type="halRef">WMT23 - Eighth Conference on Machine Translation, Dec 2023, Singapore, Singapore. pp.198--216</idno>
            <availability status="restricted">
              <licence target="https://creativecommons.org/licenses/by/4.0/">CC BY 4.0 - Attribution<ref corresp="#file-4300824-3751844"/></licence>
            </availability>
          </publicationStmt>
          <seriesStmt>
            <idno type="stamp" n="INRIA">INRIA - Institut National de Recherche en Informatique et en Automatique</idno>
            <idno type="stamp" n="INRIA-ROCQ">INRIA Paris - Rocquencourt</idno>
            <idno type="stamp" n="TESTALAIN1">TESTALAIN1</idno>
            <idno type="stamp" n="INRIA2">INRIA 2</idno>
            <idno type="stamp" n="PNRIA">Programme National de Recherche en IA</idno>
            <idno type="stamp" n="ANR">ANR</idno>
            <idno type="stamp" n="PRAIRIE-IA">PaRis AI Research InstitutE</idno>
            <idno type="stamp" n="ANR-IA-19" corresp="ANR-IA">ANR-IA-19</idno>
            <idno type="stamp" n="ANR-IA">ANR-IA</idno>
          </seriesStmt>
          <notesStmt>
            <note type="audience" n="2">International</note>
            <note type="invited" n="0">No</note>
            <note type="popular" n="0">No</note>
            <note type="peer" n="1">Yes</note>
            <note type="proceedings" n="1">Yes</note>
          </notesStmt>
          <sourceDesc>
            <biblStruct>
              <analytic>
                <title xml:lang="en">RoCS-MT: Robustness Challenge Set for Machine Translation</title>
                <author role="aut">
                  <persName>
                    <forename type="first">Rachel</forename>
                    <surname>Bawden</surname>
                  </persName>
                  <email type="md5">a08b8a5ca6961e673dd659fdfe33c881</email>
                  <email type="domain">inria.fr</email>
                  <idno type="idhal" notation="string">rachel-bawden</idno>
                  <idno type="idhal" notation="numeric">9441</idno>
                  <idno type="halauthorid" notation="string">28540-9441</idno>
                  <idno type="ORCID">https://orcid.org/0000-0001-9553-1768</idno>
                  <idno type="IDREF">https://www.idref.fr/233174591</idno>
                  <idno type="ARXIV">https://arxiv.org/a/bawden_r_1</idno>
                  <affiliation ref="#struct-482775"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Benoît</forename>
                    <surname>Sagot</surname>
                  </persName>
                  <email type="md5">ef534712ca682bf74ec7eef17d3d1b5f</email>
                  <email type="domain">inria.fr</email>
                  <idno type="idhal" notation="string">bsagot</idno>
                  <idno type="idhal" notation="numeric">1461</idno>
                  <idno type="halauthorid" notation="string">17075-1461</idno>
                  <idno type="IDREF">https://www.idref.fr/177454229</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-0107-8526</idno>
                  <idno type="ARXIV">https://arxiv.org/a/sagot_b_1</idno>
                  <idno type="GOOGLE SCHOLAR">HXUT9ZkAAAAJ</idno>
                  <affiliation ref="#struct-482775"/>
                </author>
              </analytic>
              <monogr>
                <title level="m">Proceedings of the Eighth Conference on Machine Translation</title>
                <meeting>
                  <title>WMT23 -  Eighth Conference on Machine Translation</title>
                  <date type="start">2023-12-06</date>
                  <date type="end">2023-12-07</date>
                  <settlement>Singapore</settlement>
                  <country key="SG">Singapore</country>
                </meeting>
                <imprint>
                  <biblScope unit="serie">Proceedings of the Eighth Conference on Machine Translation</biblScope>
                  <biblScope unit="pp">198--216</biblScope>
                  <date type="datePub">2023</date>
                </imprint>
              </monogr>
            </biblStruct>
          </sourceDesc>
          <profileDesc>
            <langUsage>
              <language ident="en">English</language>
            </langUsage>
            <textClass>
              <keywords scheme="author">
                <term xml:lang="en">Machine translation</term>
                <term xml:lang="en">Test suite</term>
                <term xml:lang="en">Shared task</term>
                <term xml:lang="en">Evaluation</term>
                <term xml:lang="en">Robustness</term>
                <term xml:lang="en">User-generated content</term>
                <term xml:lang="en">UGC</term>
                <term xml:lang="en">MT</term>
              </keywords>
              <classCode scheme="halDomain" n="info.info-cl">Computer Science [cs]/Computation and Language [cs.CL]</classCode>
              <classCode scheme="halTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halOldTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halTreeTypology" n="COMM">Conference papers</classCode>
            </textClass>
            <abstract xml:lang="en">
              <p>RoCS-MT, a Robust Challenge Set for Machine Translation (MT), is designed to test MT systems' ability to translate user-generated content (UGC) that displays non-standard characteristics, such as spelling errors, devowelling, acronymisation, etc. RoCS-MT is composed of English comments from Reddit, selected for their non-standard nature, which have been manually normalised and professionally translated into five languages: French, German, Czech, Ukrainian and Russian. In the context of the WMT23 test suite shared task, we analyse the models submitted to the general MT task for all from-English language pairs, offering some insights into the types of problems faced by state-of-the-art MT models when dealing with non-standard UGC texts. We compare automatic metrics for MT quality, including quality estimation to see if the same conclusions can be drawn without references. In terms of robustness, we find that many of the systems struggle with non-standard variants of words (e.g. due to phonetically inspired spellings, contraction, truncations, etc.), but that this depends on the system and the amount of training data, with the best overall systems performing better across all phenomena. GPT4 is the clear frontrunner. However we caution against drawing conclusions about generalisation capacity as it and other systems could be trained on the source side of RoCS and also on similar data.</p>
            </abstract>
          </profileDesc>
        </biblFull>
      </listBibl>
    </body>
    <back>
      <listOrg type="structures">
        <org type="researchteam" xml:id="struct-482775" status="VALID">
          <idno type="RNSR">201722248N</idno>
          <idno type="ROR">https://ror.org/05rnwy495</idno>
          <orgName>Automatic Language Modelling and ANAlysis &amp; Computational Humanities</orgName>
          <orgName type="acronym">ALMAnaCH</orgName>
          <date type="start">2017-01-01</date>
          <date type="end">2029-04-30</date>
          <desc>
            <address>
              <addrLine>48 Rue Barrault, 75013 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.inria.fr/equipes/almanach</ref>
          </desc>
          <listRelation>
            <relation active="#struct-454310" type="direct"/>
            <relation active="#struct-300009" type="indirect"/>
          </listRelation>
        </org>
        <org type="laboratory" xml:id="struct-454310" status="VALID">
          <idno type="IdRef">241614864</idno>
          <idno type="RNSR">196718247G</idno>
          <idno type="ROR">https://ror.org/05eyd5d35</idno>
          <orgName>Centre Inria de Paris</orgName>
          <date type="start">2016-03-10</date>
          <desc>
            <address>
              <addrLine>48 Rue Barrault, 75013 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/centre/paris</ref>
          </desc>
          <listRelation>
            <relation active="#struct-300009" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-300009" status="VALID">
          <idno type="ROR">https://ror.org/02kvxyf05</idno>
          <orgName>Institut National de Recherche en Informatique et en Automatique</orgName>
          <orgName type="acronym">Inria</orgName>
          <desc>
            <address>
              <addrLine>Domaine de VoluceauRocquencourt - BP 10578153 Le Chesnay Cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/en/</ref>
          </desc>
        </org>
      </listOrg>
      <listOrg type="projects">
        <org type="anrProject" xml:id="projanr-50388" status="VALID">
          <idno type="anr">ANR-19-P3IA-0001</idno>
          <orgName>PRAIRIE</orgName>
          <desc>PaRis Artificial Intelligence Research InstitutE</desc>
          <date type="start">2019</date>
        </org>
      </listOrg>
    </back>
  </text>
</TEI>