<?xml version="1.0" encoding="utf-8"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hal="http://hal.archives-ouvertes.fr/" xmlns:gml="http://www.opengis.net/gml/3.3/" xmlns:gmlce="http://www.opengis.net/gml/3.3/ce" version="1.1" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 http://api.archives-ouvertes.fr/documents/aofr-sword.xsd">
  <teiHeader>
    <fileDesc>
      <titleStmt>
        <title>HAL TEI export of hal-02431947</title>
      </titleStmt>
      <publicationStmt>
        <distributor>CCSD</distributor>
        <availability status="restricted">
          <licence target="https://creativecommons.org/publicdomain/zero/1.0/">CC0 1.0 - Universal</licence>
        </availability>
        <date when="2026-05-19T16:13:26+02:00"/>
      </publicationStmt>
      <sourceDesc>
        <p part="N">HAL API Platform</p>
      </sourceDesc>
    </fileDesc>
  </teiHeader>
  <text>
    <body>
      <listBibl>
        <biblFull>
          <titleStmt>
            <title xml:lang="en">How2: A Large-scale Dataset for Multimodal Language Understanding</title>
            <author role="aut">
              <persName>
                <forename type="first">Ramon</forename>
                <surname>Sanabria</surname>
              </persName>
              <email type="md5">0579ae0a70640387b2679e8ef7651781</email>
              <email type="domain">cs.cmu.edu</email>
              <idno type="idhal" notation="numeric">1062423</idno>
              <idno type="halauthorid" notation="string">1763745-1062423</idno>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Ozan</forename>
                <surname>Caglayan</surname>
              </persName>
              <email type="md5">2a80796e0f6b62dc346ae3959335cbaf</email>
              <email type="domain">univ-lemans.fr</email>
              <idno type="idhal" notation="string">ozan-caglayan</idno>
              <idno type="idhal" notation="numeric">16995</idno>
              <idno type="halauthorid" notation="string">33726-16995</idno>
              <idno type="ARXIV">https://arxiv.org/a/caglayan_o_1</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-5992-3470</idno>
              <idno type="GOOGLE SCHOLAR">https://scholar.google.fr/citations?user=1NTa_7QAAAAJ&amp;hl=en</idno>
              <idno type="IDREF">https://www.idref.fr/237714582</idno>
              <affiliation ref="#struct-117974"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Shruti</forename>
                <surname>Palaskar</surname>
              </persName>
              <idno type="halauthorid">1314246-0</idno>
              <affiliation ref="#struct-67135"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Desmond</forename>
                <surname>Elliott</surname>
              </persName>
              <idno type="halauthorid">742998-0</idno>
              <affiliation ref="#struct-108560"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Loïc</forename>
                <surname>Barrault</surname>
              </persName>
              <email type="md5">5dfb707b0e2ef6e50da2b895eec3feae</email>
              <email type="domain">gmail.com</email>
              <idno type="idhal" notation="string">loicbarrault</idno>
              <idno type="idhal" notation="numeric">15276</idno>
              <idno type="halauthorid" notation="string">33153-15276</idno>
              <idno type="IDREF">https://www.idref.fr/131912488</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-0634-6147</idno>
              <affiliation ref="#struct-117974"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Lucia</forename>
                <surname>Specia</surname>
              </persName>
              <idno type="halauthorid">1074052-0</idno>
              <affiliation ref="#struct-153591"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Florian</forename>
                <surname>Metze</surname>
              </persName>
              <idno type="halauthorid">356310-0</idno>
              <affiliation ref="#struct-74881"/>
            </author>
            <editor role="depositor">
              <persName>
                <forename>Loïc</forename>
                <surname>BARRAULT</surname>
              </persName>
              <email type="md5">5dfb707b0e2ef6e50da2b895eec3feae</email>
              <email type="domain">gmail.com</email>
            </editor>
          </titleStmt>
          <editionStmt>
            <edition n="v1" type="current">
              <date type="whenSubmitted">2020-01-08 11:37:15</date>
              <date type="whenModified">2026-05-19 14:10:09</date>
              <date type="whenReleased">2020-01-14 11:29:08</date>
              <date type="whenProduced">2018</date>
              <date type="whenEndEmbargoed">2020-01-08</date>
              <ref type="file" target="https://hal.science/hal-02431947v1/document">
                <date notBefore="2020-01-08"/>
              </ref>
              <ref type="file" subtype="author" n="1" target="https://hal.science/hal-02431947v1/file/26.pdf" id="file-2431947-2318781">
                <date notBefore="2020-01-08"/>
              </ref>
            </edition>
            <respStmt>
              <resp>contributor</resp>
              <name key="604178">
                <persName>
                  <forename>Loïc</forename>
                  <surname>BARRAULT</surname>
                </persName>
                <email type="md5">5dfb707b0e2ef6e50da2b895eec3feae</email>
                <email type="domain">gmail.com</email>
              </name>
            </respStmt>
          </editionStmt>
          <publicationStmt>
            <distributor>CCSD</distributor>
            <idno type="halId">hal-02431947</idno>
            <idno type="halUri">https://hal.science/hal-02431947</idno>
            <idno type="halBibtex">sanabria:hal-02431947</idno>
            <idno type="halRefHtml">&lt;i&gt;NeurIPS&lt;/i&gt;, 2018, Montréal, Canada</idno>
            <idno type="halRef">NeurIPS, 2018, Montréal, Canada</idno>
            <availability status="restricted">
              <licence target="https://about.hal.science/hal-authorisation-v1/">HAL Authorization<ref corresp="#file-2431947-2318781"/></licence>
            </availability>
          </publicationStmt>
          <seriesStmt>
            <idno type="stamp" n="UNIV-LEMANS">Université du Mans</idno>
            <idno type="stamp" n="LIUM" corresp="UNIV-LEMANS">Laboratoire Informatique de l'Université du Maine</idno>
            <idno type="stamp" n="LIUM-LST" corresp="LIUM">LIUM - Equipe Language and Speech Technology</idno>
          </seriesStmt>
          <notesStmt>
            <note type="audience" n="2">International</note>
            <note type="invited" n="0">No</note>
            <note type="popular" n="0">No</note>
            <note type="peer" n="1">Yes</note>
            <note type="proceedings" n="0">No</note>
          </notesStmt>
          <sourceDesc>
            <biblStruct>
              <analytic>
                <title xml:lang="en">How2: A Large-scale Dataset for Multimodal Language Understanding</title>
                <author role="aut">
                  <persName>
                    <forename type="first">Ramon</forename>
                    <surname>Sanabria</surname>
                  </persName>
                  <email type="md5">0579ae0a70640387b2679e8ef7651781</email>
                  <email type="domain">cs.cmu.edu</email>
                  <idno type="idhal" notation="numeric">1062423</idno>
                  <idno type="halauthorid" notation="string">1763745-1062423</idno>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Ozan</forename>
                    <surname>Caglayan</surname>
                  </persName>
                  <email type="md5">2a80796e0f6b62dc346ae3959335cbaf</email>
                  <email type="domain">univ-lemans.fr</email>
                  <idno type="idhal" notation="string">ozan-caglayan</idno>
                  <idno type="idhal" notation="numeric">16995</idno>
                  <idno type="halauthorid" notation="string">33726-16995</idno>
                  <idno type="ARXIV">https://arxiv.org/a/caglayan_o_1</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-5992-3470</idno>
                  <idno type="GOOGLE SCHOLAR">https://scholar.google.fr/citations?user=1NTa_7QAAAAJ&amp;hl=en</idno>
                  <idno type="IDREF">https://www.idref.fr/237714582</idno>
                  <affiliation ref="#struct-117974"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Shruti</forename>
                    <surname>Palaskar</surname>
                  </persName>
                  <idno type="halauthorid">1314246-0</idno>
                  <affiliation ref="#struct-67135"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Desmond</forename>
                    <surname>Elliott</surname>
                  </persName>
                  <idno type="halauthorid">742998-0</idno>
                  <affiliation ref="#struct-108560"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Loïc</forename>
                    <surname>Barrault</surname>
                  </persName>
                  <email type="md5">5dfb707b0e2ef6e50da2b895eec3feae</email>
                  <email type="domain">gmail.com</email>
                  <idno type="idhal" notation="string">loicbarrault</idno>
                  <idno type="idhal" notation="numeric">15276</idno>
                  <idno type="halauthorid" notation="string">33153-15276</idno>
                  <idno type="IDREF">https://www.idref.fr/131912488</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-0634-6147</idno>
                  <affiliation ref="#struct-117974"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Lucia</forename>
                    <surname>Specia</surname>
                  </persName>
                  <idno type="halauthorid">1074052-0</idno>
                  <affiliation ref="#struct-153591"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Florian</forename>
                    <surname>Metze</surname>
                  </persName>
                  <idno type="halauthorid">356310-0</idno>
                  <affiliation ref="#struct-74881"/>
                </author>
              </analytic>
              <monogr>
                <meeting>
                  <title>NeurIPS</title>
                  <date type="start">2018</date>
                  <settlement>Montréal</settlement>
                  <country key="CA">Canada</country>
                </meeting>
                <imprint/>
              </monogr>
            </biblStruct>
          </sourceDesc>
          <profileDesc>
            <langUsage>
              <language ident="en">English</language>
            </langUsage>
            <textClass>
              <classCode scheme="halDomain" n="info">Computer Science [cs]</classCode>
              <classCode scheme="halDomain" n="info.info-cl">Computer Science [cs]/Computation and Language [cs.CL]</classCode>
              <classCode scheme="halTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halOldTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halTreeTypology" n="COMM">Conference papers</classCode>
            </textClass>
            <abstract xml:lang="en">
              <p>Human information processing is inherently multimodal, and language is best understood in a situated context. In order to achieve human-like language processing capabilities, machines should be able to jointly process multimodal data, and not just text, images, or speech in isolation. Nevertheless, there are very few multi-modal datasets to support such research, resulting in a limited interaction among different research communities. In this paper, we introduce How2, a large-scale dataset of instructional videos covering a wide variety of topics across 80,000 clips (about 2,000 hours), with word-level time alignments to the ground-truth English subtitles. In addition to being multimodal, How2 is multilingual: we crowdsourced Portuguese translations of the subtitles. We present results for monomodal and multimodal baselines on several language processing tasks with interesting insights on the utility of different modalities. We hope that by making the How2 dataset and baselines available we will encourage collaboration across language, speech and vision communities.</p>
            </abstract>
          </profileDesc>
        </biblFull>
      </listBibl>
    </body>
    <back>
      <listOrg type="structures">
        <org type="laboratory" xml:id="struct-117974" status="VALID">
          <idno type="IdRef">035240776</idno>
          <idno type="RNSR">200615275G</idno>
          <idno type="ROR">https://ror.org/03gvgnv89</idno>
          <idno type="Wikidata">Q51779666</idno>
          <orgName>Laboratoire d'Informatique de l'Université du Mans</orgName>
          <orgName type="acronym">LIUM</orgName>
          <desc>
            <address>
              <addrLine>Avenue Laennec 72085 Le Mans cedex 9</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://lium.univ-lemans.fr/</ref>
          </desc>
          <listRelation>
            <relation name="EA4023" active="#struct-7566" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-67135" status="VALID">
          <orgName>Carnegie Mellon University [Pittsburgh]</orgName>
          <orgName type="acronym">CMU</orgName>
          <desc>
            <address>
              <addrLine>5000 Forbes Ave,  Pittsburgh, PA 15213</addrLine>
              <country key="US"/>
            </address>
            <ref type="url">http://www.cmu.edu/</ref>
          </desc>
        </org>
        <org type="laboratory" xml:id="struct-108560" status="VALID">
          <orgName>Department of Computing Science</orgName>
          <desc>
            <address>
              <country key="GB"/>
            </address>
          </desc>
          <listRelation>
            <relation active="#struct-25887" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-153591" status="VALID">
          <idno type="ROR">https://ror.org/05krs5044</idno>
          <orgName>University of Sheffield [Sheffield]</orgName>
          <desc>
            <address>
              <addrLine>Western Bank Sheffield S10 2TN</addrLine>
              <country key="GB"/>
            </address>
            <ref type="url">http://www.sheffield.ac.uk/</ref>
          </desc>
        </org>
        <org type="laboratory" xml:id="struct-74881" status="VALID">
          <orgName>Interactive Systems Labs</orgName>
          <orgName type="acronym">ISL</orgName>
          <desc>
            <address>
              <addrLine>Universität Karlsruhe (TH)</addrLine>
              <country key="DE"/>
            </address>
            <ref type="url">http://isl.ira.uka.de/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-301792" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-7566" status="VALID">
          <idno type="IdRef">026404435</idno>
          <idno type="ROR">https://ror.org/01mtcc283</idno>
          <orgName>Le Mans Université</orgName>
          <orgName type="acronym">UM</orgName>
          <desc>
            <address>
              <addrLine>Avenue Olivier Messiaen - 72085 Le Mans cedex 9</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.univ-lemans.fr/</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-25887" status="VALID">
          <idno type="IdRef">02643184X</idno>
          <idno type="ISNI">000000012193314X</idno>
          <idno type="ROR">https://ror.org/00vtgdb53</idno>
          <orgName>University of Glasgow</orgName>
          <date type="start">1451-01-01</date>
          <desc>
            <address>
              <addrLine>Glasgow G12 8QQ Scotland</addrLine>
              <country key="GB"/>
            </address>
            <ref type="url">https://www.gla.ac.uk/</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-301792" status="VALID">
          <orgName>Universität Karlsruhe</orgName>
          <orgName type="acronym">TH</orgName>
          <desc>
            <address>
              <addrLine>Universität Karlsruhe (TH) 76128 Karlsruhe</addrLine>
              <country key="DE"/>
            </address>
            <ref type="url">http://www.uni-karlsruhe.de</ref>
          </desc>
        </org>
      </listOrg>
    </back>
  </text>
</TEI>