<?xml version="1.0" encoding="utf-8"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hal="http://hal.archives-ouvertes.fr/" xmlns:gml="http://www.opengis.net/gml/3.3/" xmlns:gmlce="http://www.opengis.net/gml/3.3/ce" version="1.1" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 http://api.archives-ouvertes.fr/documents/aofr-sword.xsd">
  <teiHeader>
    <fileDesc>
      <titleStmt>
        <title>HAL TEI export of hal-04509124</title>
      </titleStmt>
      <publicationStmt>
        <distributor>CCSD</distributor>
        <availability status="restricted">
          <licence target="https://creativecommons.org/publicdomain/zero/1.0/">CC0 1.0 - Universal</licence>
        </availability>
        <date when="2026-05-12T10:24:46+02:00"/>
      </publicationStmt>
      <sourceDesc>
        <p part="N">HAL API Platform</p>
      </sourceDesc>
    </fileDesc>
  </teiHeader>
  <text>
    <body>
      <listBibl>
        <biblFull>
          <titleStmt>
            <title xml:lang="en">Semi-supervised Multimodal Representation Learning through a Global Workspace</title>
            <author role="aut">
              <persName>
                <forename type="first">Benjamin</forename>
                <surname>Devillers</surname>
              </persName>
              <idno type="halauthorid">1688468-0</idno>
              <affiliation ref="#struct-1149092"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Léopold</forename>
                <surname>Maytié</surname>
              </persName>
              <idno type="halauthorid">3089303-0</idno>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Rufin</forename>
                <surname>VanRullen</surname>
              </persName>
              <idno type="halauthorid">2783877-0</idno>
              <affiliation ref="#struct-1149092"/>
            </author>
            <editor role="depositor">
              <persName>
                <forename>Rufin</forename>
                <surname>VANRULLEN</surname>
              </persName>
              <email type="md5">057fd3a0feadf8a5efb40c99a9e4b690</email>
              <email type="domain">cnrs.fr</email>
            </editor>
            <funder ref="#projanr-50372"/>
            <funder ref="#projeurop-729218"/>
          </titleStmt>
          <editionStmt>
            <edition n="v1" type="current">
              <date type="whenSubmitted">2024-03-18 11:58:55</date>
              <date type="whenModified">2025-10-22 18:04:09</date>
              <date type="whenReleased">2024-03-18 11:58:55</date>
              <date type="whenProduced">2024-03-18</date>
              <ref type="externalLink" target="http://arxiv.org/pdf/2306.15711"/>
            </edition>
            <respStmt>
              <resp>contributor</resp>
              <name key="368849">
                <persName>
                  <forename>Rufin</forename>
                  <surname>VANRULLEN</surname>
                </persName>
                <email type="md5">057fd3a0feadf8a5efb40c99a9e4b690</email>
                <email type="domain">cnrs.fr</email>
              </name>
            </respStmt>
          </editionStmt>
          <publicationStmt>
            <distributor>CCSD</distributor>
            <idno type="halId">hal-04509124</idno>
            <idno type="halUri">https://hal.science/hal-04509124</idno>
            <idno type="halBibtex">devillers:hal-04509124</idno>
            <idno type="halRefHtml">2024</idno>
            <idno type="halRef">2024</idno>
            <availability status="restricted"/>
          </publicationStmt>
          <seriesStmt>
            <idno type="stamp" n="UNIV-TLSE2">Université Toulouse 2</idno>
            <idno type="stamp" n="UNIV-TLSE3">Université de Toulouse</idno>
            <idno type="stamp" n="CNRS">CNRS - Centre national de la recherche scientifique</idno>
            <idno type="stamp" n="CERCO">Centre de Recherche Cerveau et Cognition</idno>
            <idno type="stamp" n="OPENAIRE">OpenAIRE</idno>
            <idno type="stamp" n="PNRIA">Programme National de Recherche en IA</idno>
            <idno type="stamp" n="ANR">ANR</idno>
            <idno type="stamp" n="ANITI">ANITI - Artificial and Natural Intelligence Toulouse Institute</idno>
            <idno type="stamp" n="UNIV-UT3">Université Toulouse 3</idno>
            <idno type="stamp" n="UT3-INP">Université de Toulouse / Toulouse INP</idno>
            <idno type="stamp" n="UT3-TOULOUSEINP">Université de Toulouse / Toulouse INP</idno>
            <idno type="stamp" n="ANR-IA-19" corresp="ANR-IA">ANR-IA-19</idno>
            <idno type="stamp" n="ANR-IA">ANR-IA</idno>
          </seriesStmt>
          <notesStmt>
            <note type="commentary">Under review</note>
          </notesStmt>
          <sourceDesc>
            <biblStruct>
              <analytic>
                <title xml:lang="en">Semi-supervised Multimodal Representation Learning through a Global Workspace</title>
                <author role="aut">
                  <persName>
                    <forename type="first">Benjamin</forename>
                    <surname>Devillers</surname>
                  </persName>
                  <idno type="halauthorid">1688468-0</idno>
                  <affiliation ref="#struct-1149092"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Léopold</forename>
                    <surname>Maytié</surname>
                  </persName>
                  <idno type="halauthorid">3089303-0</idno>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Rufin</forename>
                    <surname>VanRullen</surname>
                  </persName>
                  <idno type="halauthorid">2783877-0</idno>
                  <affiliation ref="#struct-1149092"/>
                </author>
              </analytic>
              <monogr>
                <imprint>
                  <date type="datePub">2023-06-27</date>
                </imprint>
              </monogr>
              <idno type="arxiv">2306.15711</idno>
              <idno type="doi">10.48550/arXiv.2306.15711</idno>
            </biblStruct>
          </sourceDesc>
          <profileDesc>
            <langUsage>
              <language ident="en">English</language>
            </langUsage>
            <textClass>
              <classCode scheme="halDomain" n="info.info-ai">Computer Science [cs]/Artificial Intelligence [cs.AI]</classCode>
              <classCode scheme="halTypology" n="UNDEFINED">Preprints, Working Papers, ...</classCode>
              <classCode scheme="halOldTypology" n="UNDEFINED">Preprints, Working Papers, ...</classCode>
              <classCode scheme="halTreeTypology" n="UNDEFINED">Preprints, Working Papers, ...</classCode>
            </textClass>
            <abstract xml:lang="en">
              <p>Recent deep learning models can efficiently combine inputs from different modalities (e.g., images and text) and learn to align their latent representations, or to translate signals from one domain to another (as in image captioning, or text-to-image generation). However, current approaches mainly rely on brute-force supervised training over large multimodal datasets. In contrast, humans (and other animals) can learn useful multimodal representations from only sparse experience with matched cross-modal data. Here we evaluate the capabilities of a neural network architecture inspired by the cognitive notion of a "Global Workspace": a shared representation for two (or more) input modalities. Each modality is processed by a specialized system (pretrained on unimodal data, and subsequently frozen). The corresponding latent representations are then encoded to and decoded from a single shared workspace. Importantly, this architecture is amenable to self-supervised training via cycle-consistency: encoding-decoding sequences should approximate the identity function. For various pairings of vision-language modalities and across two datasets of varying complexity, we show that such an architecture can be trained to align and translate between two modalities with very little need for matched data (from 4 to 7 times less than a fully supervised approach). The global workspace representation can be used advantageously for downstream classification tasks and for robust transfer learning. Ablation studies reveal that both the shared workspace and the self-supervised cycle-consistency training are critical to the system's performance.</p>
            </abstract>
          </profileDesc>
        </biblFull>
      </listBibl>
    </body>
    <back>
      <listOrg type="structures">
        <org type="laboratory" xml:id="struct-1149092" status="OLD">
          <idno type="IdRef">179351788</idno>
          <idno type="ISNI">0000 0000 8523 0913</idno>
          <idno type="RNSR">199311991U</idno>
          <idno type="ROR">https://ror.org/04fhrs205</idno>
          <orgName>Centre de recherche cerveau et cognition</orgName>
          <orgName type="acronym">CERCO UMR5549</orgName>
          <date type="start">2021-01-01</date>
          <date type="end">2025-01-01</date>
          <desc>
            <address>
              <addrLine>CHU Purpan - Pavillon Baudot BP 25202 31052 Toulouse Cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://cerco.cnrs.fr/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-217752" type="direct"/>
            <relation active="#struct-443875" type="indirect"/>
            <relation name="UMR5549" active="#struct-441569" type="direct"/>
            <relation active="#struct-1149079" type="direct"/>
            <relation active="#struct-116256" type="indirect"/>
            <relation name="FED4171" active="#struct-217752" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-217752" status="OLD">
          <idno type="IdRef">026404672</idno>
          <idno type="ISNI">0000000121617331</idno>
          <idno type="ROR">https://ror.org/02v6kpv12</idno>
          <idno type="Wikidata">Q1273188</idno>
          <orgName>Université Toulouse III - Paul Sabatier</orgName>
          <orgName type="acronym">UT3</orgName>
          <date type="end">2025-01-01</date>
          <desc>
            <address>
              <addrLine>118 route de Narbonne - 31062 Toulouse</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.univ-tlse3.fr/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-443875" type="direct"/>
          </listRelation>
        </org>
        <org type="regroupinstitution" xml:id="struct-443875" status="VALID">
          <idno type="ROR">https://ror.org/017tgbk05</idno>
          <orgName>Communauté d'universités et établissements de Toulouse</orgName>
          <orgName type="acronym">Comue de Toulouse</orgName>
          <desc>
            <address>
              <addrLine>41 Allée Jules Guesde, 31000 Toulouse</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.univ-toulouse.fr/</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-441569" status="VALID">
          <idno type="IdRef">02636817X</idno>
          <idno type="ISNI">0000000122597504</idno>
          <idno type="ROR">https://ror.org/02feahw73</idno>
          <orgName>Centre National de la Recherche Scientifique</orgName>
          <orgName type="acronym">CNRS</orgName>
          <date type="start">1939-10-19</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.cnrs.fr/</ref>
          </desc>
        </org>
        <org type="regrouplaboratory" xml:id="struct-1149079" status="OLD">
          <orgName>Toulouse Mind &amp; Brain Institut</orgName>
          <orgName type="acronym">TMBI</orgName>
          <date type="start">2021-01-01</date>
          <date type="end">2025-01-01</date>
          <desc>
            <address>
              <addrLine>CHU Purpan, Pavillon Baudot, 31059 Toulouse cedex 9</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.tmbi.fr/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-116256" type="direct"/>
            <relation active="#struct-443875" type="indirect"/>
            <relation name="FED4171" active="#struct-217752" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-116256" status="VALID">
          <idno type="IdRef">026403994</idno>
          <idno type="ROR">https://ror.org/04ezk3x31</idno>
          <orgName>Université Toulouse - Jean Jaurès</orgName>
          <orgName type="acronym">UT2J</orgName>
          <date type="start">1971-01-01</date>
          <desc>
            <address>
              <addrLine>5 allées Antonio Machado - 31058 Toulouse Cedex 9</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.univ-tlse2.fr</ref>
          </desc>
          <listRelation>
            <relation active="#struct-443875" type="direct"/>
          </listRelation>
        </org>
      </listOrg>
      <listOrg type="projects">
        <org type="anrProject" xml:id="projanr-50372" status="VALID">
          <idno type="anr">ANR-19-P3IA-0004</idno>
          <orgName>ANITI</orgName>
          <desc>Artificial and Natural Intelligence Toulouse Institute</desc>
          <date type="start">2019</date>
        </org>
        <org type="europeanProject" xml:id="projeurop-729218" status="INCOMING">
          <orgName>ERC GLOW</orgName>
          <desc>ERC Advanced Grant No 101096017. The Global Latent Workspace: towards AI models of flexible cognition</desc>
        </org>
      </listOrg>
    </back>
  </text>
</TEI>