<?xml version="1.0" encoding="utf-8"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hal="http://hal.archives-ouvertes.fr/" xmlns:gml="http://www.opengis.net/gml/3.3/" xmlns:gmlce="http://www.opengis.net/gml/3.3/ce" version="1.1" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 http://api.archives-ouvertes.fr/documents/aofr-sword.xsd">
  <teiHeader>
    <fileDesc>
      <titleStmt>
        <title>HAL TEI export of hal-04366059</title>
      </titleStmt>
      <publicationStmt>
        <distributor>CCSD</distributor>
        <availability status="restricted">
          <licence target="https://creativecommons.org/publicdomain/zero/1.0/">CC0 1.0 - Universal</licence>
        </availability>
        <date when="2026-05-02T09:14:11+02:00"/>
      </publicationStmt>
      <sourceDesc>
        <p part="N">HAL API Platform</p>
      </sourceDesc>
    </fileDesc>
  </teiHeader>
  <text>
    <body>
      <listBibl>
        <biblFull>
          <titleStmt>
            <title xml:lang="en">UnIVAL: Unified Model for Image, Video, Audio and Language Tasks</title>
            <author role="aut">
              <persName>
                <forename type="first">Mustafa</forename>
                <surname>Shukor</surname>
              </persName>
              <email type="md5">24dd60f231d8984dd0c68a7876b445a5</email>
              <email type="domain">isir.upmc.fr</email>
              <ptr type="url" target="https://scholar.google.com/citations?user=lhp9mRgAAAAJ&amp;hl=en"/>
              <idno type="idhal" notation="string">mustafa-shukor</idno>
              <idno type="idhal" notation="numeric">1173556</idno>
              <idno type="halauthorid" notation="string">2610070-1173556</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-3100-6689</idno>
              <affiliation ref="#struct-1085231"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Corentin</forename>
                <surname>Dancette</surname>
              </persName>
              <idno type="halauthorid">1433337-0</idno>
              <affiliation ref="#struct-1085231"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Alexandre</forename>
                <surname>Rame</surname>
              </persName>
              <idno type="halauthorid">2670242-0</idno>
              <affiliation ref="#struct-1085231"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Matthieu</forename>
                <surname>Cord</surname>
              </persName>
              <idno type="halauthorid">13283-0</idno>
              <affiliation ref="#struct-1085231"/>
            </author>
            <editor role="depositor">
              <persName>
                <forename>Mustafa</forename>
                <surname>Shukor</surname>
              </persName>
              <email type="md5">24dd60f231d8984dd0c68a7876b445a5</email>
              <email type="domain">isir.upmc.fr</email>
            </editor>
            <funder ref="#projanr-51639"/>
          </titleStmt>
          <editionStmt>
            <edition n="v1" type="current">
              <date type="whenSubmitted">2023-12-28 18:41:05</date>
              <date type="whenModified">2025-02-07 18:37:53</date>
              <date type="whenReleased">2024-01-02 10:55:18</date>
              <date type="whenProduced">2023-12</date>
              <date type="whenEndEmbargoed">2023-12-28</date>
              <ref type="file" target="https://hal.science/hal-04366059v1/document">
                <date notBefore="2023-12-28"/>
              </ref>
              <ref type="file" subtype="author" n="1" target="https://hal.science/hal-04366059v1/file/2307.16184.pdf" id="file-4366059-3799589">
                <date notBefore="2023-12-28"/>
              </ref>
              <ref type="externalLink" target="http://arxiv.org/pdf/2307.16184"/>
            </edition>
            <respStmt>
              <resp>contributor</resp>
              <name key="1328722">
                <persName>
                  <forename>Mustafa</forename>
                  <surname>Shukor</surname>
                </persName>
                <email type="md5">24dd60f231d8984dd0c68a7876b445a5</email>
                <email type="domain">isir.upmc.fr</email>
              </name>
            </respStmt>
          </editionStmt>
          <publicationStmt>
            <distributor>CCSD</distributor>
            <idno type="halId">hal-04366059</idno>
            <idno type="halUri">https://hal.science/hal-04366059</idno>
            <idno type="halBibtex">shukor:hal-04366059</idno>
            <idno type="halRefHtml">&lt;i&gt;Transactions on Machine Learning Research Journal&lt;/i&gt;, 2023</idno>
            <idno type="halRef">Transactions on Machine Learning Research Journal, 2023</idno>
            <availability status="restricted">
              <licence target="https://about.hal.science/hal-authorisation-v1/">HAL Authorization<ref corresp="#file-4366059-3799589"/></licence>
            </availability>
          </publicationStmt>
          <seriesStmt>
            <idno type="stamp" n="CNRS">CNRS - Centre national de la recherche scientifique</idno>
            <idno type="stamp" n="ISIR" corresp="SORBONNE-UNIVERSITE">Institut des Systèmes Intelligents et de Robotique</idno>
            <idno type="stamp" n="GENCI">Publications ayant eu recours aux supercalculateurs du GENCI</idno>
            <idno type="stamp" n="SORBONNE-UNIVERSITE">Sorbonne Université</idno>
            <idno type="stamp" n="SORBONNE-UNIV" corresp="SORBONNE-UNIVERSITE">Sorbonne Université 01/01/2018</idno>
            <idno type="stamp" n="SU-SCIENCES" corresp="SORBONNE-UNIVERSITE">Faculté des Sciences de Sorbonne Université</idno>
            <idno type="stamp" n="SU-TI">Sorbonne Université - Texte Intégral</idno>
            <idno type="stamp" n="ANR">ANR</idno>
            <idno type="stamp" n="ALLIANCE-SU"> Alliance Sorbonne Université</idno>
            <idno type="stamp" n="SUPRA_INGENIERIE">Ingénierie</idno>
            <idno type="stamp" n="ANR-IA-20" corresp="ANR-IA">ANR-IA-20</idno>
            <idno type="stamp" n="ANR-IA">ANR-IA</idno>
          </seriesStmt>
          <notesStmt>
            <note type="commentary">Accepted at TMLR 2023. 40 pages. Project page: https://unival-model.github.io/</note>
            <note type="audience" n="2">International</note>
            <note type="popular" n="0">No</note>
            <note type="peer" n="1">Yes</note>
          </notesStmt>
          <sourceDesc>
            <biblStruct>
              <analytic>
                <title xml:lang="en">UnIVAL: Unified Model for Image, Video, Audio and Language Tasks</title>
                <author role="aut">
                  <persName>
                    <forename type="first">Mustafa</forename>
                    <surname>Shukor</surname>
                  </persName>
                  <email type="md5">24dd60f231d8984dd0c68a7876b445a5</email>
                  <email type="domain">isir.upmc.fr</email>
                  <ptr type="url" target="https://scholar.google.com/citations?user=lhp9mRgAAAAJ&amp;hl=en"/>
                  <idno type="idhal" notation="string">mustafa-shukor</idno>
                  <idno type="idhal" notation="numeric">1173556</idno>
                  <idno type="halauthorid" notation="string">2610070-1173556</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-3100-6689</idno>
                  <affiliation ref="#struct-1085231"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Corentin</forename>
                    <surname>Dancette</surname>
                  </persName>
                  <idno type="halauthorid">1433337-0</idno>
                  <affiliation ref="#struct-1085231"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Alexandre</forename>
                    <surname>Rame</surname>
                  </persName>
                  <idno type="halauthorid">2670242-0</idno>
                  <affiliation ref="#struct-1085231"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Matthieu</forename>
                    <surname>Cord</surname>
                  </persName>
                  <idno type="halauthorid">13283-0</idno>
                  <affiliation ref="#struct-1085231"/>
                </author>
              </analytic>
              <monogr>
                <idno type="halJournalId" status="VALID">363201</idno>
                <idno type="issn">2835-8856</idno>
                <title level="j">Transactions on Machine Learning Research Journal</title>
                <imprint>
                  <publisher>[Amherst Massachusetts]: OpenReview.net, 2022</publisher>
                  <date type="datePub">2023-12</date>
                </imprint>
              </monogr>
              <idno type="arxiv">2307.16184</idno>
            </biblStruct>
          </sourceDesc>
          <profileDesc>
            <langUsage>
              <language ident="en">English</language>
            </langUsage>
            <textClass>
              <keywords scheme="author">
                <term xml:lang="en">Foundation models</term>
                <term xml:lang="en">Multimodal models</term>
              </keywords>
              <classCode scheme="halDomain" n="info.info-cv">Computer Science [cs]/Computer Vision and Pattern Recognition [cs.CV]</classCode>
              <classCode scheme="halDomain" n="info.info-lg">Computer Science [cs]/Machine Learning [cs.LG]</classCode>
              <classCode scheme="halTypology" n="ART">Journal articles</classCode>
              <classCode scheme="halOldTypology" n="ART">Journal articles</classCode>
              <classCode scheme="halTreeTypology" n="ART">Journal articles</classCode>
            </textClass>
            <abstract xml:lang="en">
              <p>Large Language Models (LLMs) have made the ambitious quest for generalist agents significantly far from being a fantasy. A key hurdle for building such general models is the diversity and heterogeneity of tasks and modalities. A promising solution is unification, allowing the support of a myriad of tasks and modalities within one unified framework. While few large models (e.g., Flamingo (Alayrac et al., 2022)), trained on massive datasets, can support more than two modalities, current small to mid-scale unified models are still limited to 2 modalities, usually image-text or video-text. The question that we ask is: is it possible to build efficiently a unified model that can support all modalities? To answer this, we propose UnIVAL, a step further towards this ambitious goal. Without relying on fancy datasets sizes or models with billions of parameters, the ∼ 0.25B parameter UnIVAL model goes beyond two modalities and unifies text, images, video, and audio into a single model. Our model is efficiently pretrained on many tasks, based on task balancing and multimodal curriculum learning. UnIVAL shows competitive performance to existing state-of-the-art approaches, across image and video-text tasks. The feature representations learned from image and video-text modalities, allows the model to achieve competitive performance when finetuned on audio-text tasks, despite not being pretrained on audio. Thanks to the unified model, we propose a novel study on multimodal model merging via weight interpolation of models trained on different multimodal tasks, showing their benefits in particular for out-ofdistribution generalization. Finally, we motivate unification by showing the synergy between tasks. The model weights and code are available at: https://github.com/mshukor/UnIVAL.</p>
            </abstract>
          </profileDesc>
        </biblFull>
      </listBibl>
    </body>
    <back>
      <listOrg type="structures">
        <org type="researchteam" xml:id="struct-1085231" status="VALID">
          <orgName>Machine Learning and Information Access</orgName>
          <orgName type="acronym">MLIA</orgName>
          <date type="start">2022-01-01</date>
          <desc>
            <address>
              <addrLine>Tour 65-66 3ème étage4, place Jussieu75005 Paris – France</addrLine>
              <country key="FR"/>
            </address>
          </desc>
          <listRelation>
            <relation active="#struct-541937" type="direct"/>
            <relation active="#struct-413221" type="indirect"/>
            <relation name="UMR7222" active="#struct-441569" type="indirect"/>
          </listRelation>
        </org>
        <org type="laboratory" xml:id="struct-541937" status="VALID">
          <idno type="IdRef">241179122</idno>
          <idno type="RNSR">200918463J</idno>
          <idno type="ROR">05neq8668</idno>
          <orgName>Institut des Systèmes Intelligents et de Robotique</orgName>
          <orgName type="acronym">ISIR</orgName>
          <date type="start">2018-01-01</date>
          <desc>
            <address>
              <addrLine>Sorbonne-Université, Boite courrier 173 4 Place JUSSIEU 75252 Paris cedex 05</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.isir.upmc.fr</ref>
          </desc>
          <listRelation>
            <relation active="#struct-413221" type="direct"/>
            <relation name="UMR7222" active="#struct-441569" type="direct"/>
          </listRelation>
        </org>
        <org type="regroupinstitution" xml:id="struct-413221" status="VALID">
          <idno type="IdRef">221333754</idno>
          <idno type="ROR">https://ror.org/02en5vm52</idno>
          <orgName>Sorbonne Université</orgName>
          <orgName type="acronym">SU</orgName>
          <date type="start">2018-01-01</date>
          <desc>
            <address>
              <addrLine>21 rue de l’École de médecine - 75006 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.sorbonne-universite.fr/</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-441569" status="VALID">
          <idno type="IdRef">02636817X</idno>
          <idno type="ISNI">0000000122597504</idno>
          <idno type="ROR">https://ror.org/02feahw73</idno>
          <orgName>Centre National de la Recherche Scientifique</orgName>
          <orgName type="acronym">CNRS</orgName>
          <date type="start">1939-10-19</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.cnrs.fr/</ref>
          </desc>
        </org>
      </listOrg>
      <listOrg type="projects">
        <org type="anrProject" xml:id="projanr-51639" status="VALID">
          <idno type="anr">ANR-20-CHIA-0022</idno>
          <orgName>VISA DEEP</orgName>
          <desc>Raisonnement Visuel dans les architectures de réseaux de neurones profonds</desc>
          <date type="start">2020</date>
        </org>
      </listOrg>
    </back>
  </text>
</TEI>