<?xml version="1.0" encoding="utf-8"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hal="http://hal.archives-ouvertes.fr/" xmlns:gml="http://www.opengis.net/gml/3.3/" xmlns:gmlce="http://www.opengis.net/gml/3.3/ce" version="1.1" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 http://api.archives-ouvertes.fr/documents/aofr-sword.xsd">
  <teiHeader>
    <fileDesc>
      <titleStmt>
        <title>HAL TEI export of hal-02912027</title>
      </titleStmt>
      <publicationStmt>
        <distributor>CCSD</distributor>
        <availability status="restricted">
          <licence target="https://creativecommons.org/publicdomain/zero/1.0/">CC0 1.0 - Universal</licence>
        </availability>
        <date when="2026-05-19T16:13:13+02:00"/>
      </publicationStmt>
      <sourceDesc>
        <p part="N">HAL API Platform</p>
      </sourceDesc>
    </fileDesc>
  </teiHeader>
  <text>
    <body>
      <listBibl>
        <biblFull>
          <titleStmt>
            <title xml:lang="en">Robust Training of Vector Quantized Bottleneck Models</title>
            <author role="aut">
              <persName>
                <forename type="first">Adrian</forename>
                <surname>Łańcucki</surname>
              </persName>
              <idno type="halauthorid">1739584-0</idno>
              <affiliation ref="#struct-305685"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Jan</forename>
                <surname>Chorowski</surname>
              </persName>
              <idno type="halauthorid">1739582-0</idno>
              <affiliation ref="#struct-305685"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Guillaume</forename>
                <surname>Sanchez</surname>
              </persName>
              <email type="md5">1f89cd6a94e4a1e5fea6767a78ba5a0c</email>
              <email type="domain">gmail.com</email>
              <idno type="idhal" notation="numeric">1068521</idno>
              <idno type="halauthorid" notation="string">498216-1068521</idno>
              <affiliation ref="#struct-530702"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Ricard</forename>
                <surname>Marxer</surname>
              </persName>
              <email type="md5">5acbab6a8b091e3c949098a46a2ba92b</email>
              <email type="domain">univ-tln.fr</email>
              <idno type="idhal" notation="string">ricard-marxer</idno>
              <idno type="idhal" notation="numeric">19391</idno>
              <idno type="halauthorid" notation="string">27165-19391</idno>
              <idno type="ORCID">https://orcid.org/0000-0001-5099-5059</idno>
              <idno type="IDREF">https://www.idref.fr/240437713</idno>
              <affiliation ref="#struct-527033"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Nanxin</forename>
                <surname>Chen</surname>
              </persName>
              <idno type="halauthorid">1096285-0</idno>
              <affiliation ref="#struct-300813"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Hans</forename>
                <forename type="middle">J G A</forename>
                <surname>Dolfing</surname>
              </persName>
              <idno type="halauthorid">1739583-0</idno>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Sameer</forename>
                <surname>Khurana</surname>
              </persName>
              <idno type="halauthorid">2008278-0</idno>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Tanel</forename>
                <surname>Alumäe</surname>
              </persName>
              <idno type="halauthorid">1739585-0</idno>
              <affiliation ref="#struct-310135"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Antoine</forename>
                <surname>Laurent</surname>
              </persName>
              <email type="md5">58703a365f961c551581ea38233d8ddc</email>
              <email type="domain">univ-lemans.fr</email>
              <idno type="idhal" notation="string">antoine-laurent</idno>
              <idno type="idhal" notation="numeric">13586</idno>
              <idno type="halauthorid" notation="string">33154-13586</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-2653-1008</idno>
              <idno type="IDREF">https://www.idref.fr/147099072</idno>
              <affiliation ref="#struct-117974"/>
            </author>
            <editor role="depositor">
              <persName>
                <forename>Antoine</forename>
                <surname>LAURENT</surname>
              </persName>
              <email type="md5">58703a365f961c551581ea38233d8ddc</email>
              <email type="domain">univ-lemans.fr</email>
            </editor>
          </titleStmt>
          <editionStmt>
            <edition n="v1" type="current">
              <date type="whenSubmitted">2020-08-05 09:41:37</date>
              <date type="whenModified">2026-05-19 14:10:09</date>
              <date type="whenReleased">2020-08-05 10:37:35</date>
              <date type="whenProduced">2020-07-19</date>
              <date type="whenEndEmbargoed">2020-08-05</date>
              <ref type="file" target="https://hal.science/hal-02912027v1/document">
                <date notBefore="2020-08-05"/>
              </ref>
              <ref type="file" subtype="author" n="1" target="https://hal.science/hal-02912027v1/file/robust_vq_arxiv.pdf" id="file-2912027-2564007">
                <date notBefore="2020-08-05"/>
              </ref>
            </edition>
            <respStmt>
              <resp>contributor</resp>
              <name key="492760">
                <persName>
                  <forename>Antoine</forename>
                  <surname>LAURENT</surname>
                </persName>
                <email type="md5">58703a365f961c551581ea38233d8ddc</email>
                <email type="domain">univ-lemans.fr</email>
              </name>
            </respStmt>
          </editionStmt>
          <publicationStmt>
            <distributor>CCSD</distributor>
            <idno type="halId">hal-02912027</idno>
            <idno type="halUri">https://hal.science/hal-02912027</idno>
            <idno type="halBibtex">acucki:hal-02912027</idno>
            <idno type="halRefHtml">&lt;i&gt;IJCNN 2020&lt;/i&gt;, Jul 2020, Glasgow, United Kingdom</idno>
            <idno type="halRef">IJCNN 2020, Jul 2020, Glasgow, United Kingdom</idno>
            <availability status="restricted">
              <licence target="https://about.hal.science/hal-authorisation-v1/">HAL Authorization<ref corresp="#file-2912027-2564007"/></licence>
            </availability>
          </publicationStmt>
          <seriesStmt>
            <idno type="stamp" n="UNIV-TLN">Université de Toulon</idno>
            <idno type="stamp" n="CNRS">CNRS - Centre national de la recherche scientifique</idno>
            <idno type="stamp" n="UNIV-AMU">Aix Marseille Université</idno>
            <idno type="stamp" n="UNIV-LEMANS">Université du Mans</idno>
            <idno type="stamp" n="LIUM" corresp="UNIV-LEMANS">Laboratoire Informatique de l'Université du Maine</idno>
            <idno type="stamp" n="LIS-LAB">Laboratoire d'Informatique et Systèmes</idno>
          </seriesStmt>
          <notesStmt>
            <note type="commentary">Virtual Conference</note>
            <note type="audience" n="2">International</note>
            <note type="invited" n="0">No</note>
            <note type="popular" n="0">No</note>
            <note type="peer" n="1">Yes</note>
            <note type="proceedings" n="0">No</note>
          </notesStmt>
          <sourceDesc>
            <biblStruct>
              <analytic>
                <title xml:lang="en">Robust Training of Vector Quantized Bottleneck Models</title>
                <author role="aut">
                  <persName>
                    <forename type="first">Adrian</forename>
                    <surname>Łańcucki</surname>
                  </persName>
                  <idno type="halauthorid">1739584-0</idno>
                  <affiliation ref="#struct-305685"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Jan</forename>
                    <surname>Chorowski</surname>
                  </persName>
                  <idno type="halauthorid">1739582-0</idno>
                  <affiliation ref="#struct-305685"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Guillaume</forename>
                    <surname>Sanchez</surname>
                  </persName>
                  <email type="md5">1f89cd6a94e4a1e5fea6767a78ba5a0c</email>
                  <email type="domain">gmail.com</email>
                  <idno type="idhal" notation="numeric">1068521</idno>
                  <idno type="halauthorid" notation="string">498216-1068521</idno>
                  <affiliation ref="#struct-530702"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Ricard</forename>
                    <surname>Marxer</surname>
                  </persName>
                  <email type="md5">5acbab6a8b091e3c949098a46a2ba92b</email>
                  <email type="domain">univ-tln.fr</email>
                  <idno type="idhal" notation="string">ricard-marxer</idno>
                  <idno type="idhal" notation="numeric">19391</idno>
                  <idno type="halauthorid" notation="string">27165-19391</idno>
                  <idno type="ORCID">https://orcid.org/0000-0001-5099-5059</idno>
                  <idno type="IDREF">https://www.idref.fr/240437713</idno>
                  <affiliation ref="#struct-527033"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Nanxin</forename>
                    <surname>Chen</surname>
                  </persName>
                  <idno type="halauthorid">1096285-0</idno>
                  <affiliation ref="#struct-300813"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Hans</forename>
                    <forename type="middle">J G A</forename>
                    <surname>Dolfing</surname>
                  </persName>
                  <idno type="halauthorid">1739583-0</idno>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Sameer</forename>
                    <surname>Khurana</surname>
                  </persName>
                  <idno type="halauthorid">2008278-0</idno>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Tanel</forename>
                    <surname>Alumäe</surname>
                  </persName>
                  <idno type="halauthorid">1739585-0</idno>
                  <affiliation ref="#struct-310135"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Antoine</forename>
                    <surname>Laurent</surname>
                  </persName>
                  <email type="md5">58703a365f961c551581ea38233d8ddc</email>
                  <email type="domain">univ-lemans.fr</email>
                  <idno type="idhal" notation="string">antoine-laurent</idno>
                  <idno type="idhal" notation="numeric">13586</idno>
                  <idno type="halauthorid" notation="string">33154-13586</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-2653-1008</idno>
                  <idno type="IDREF">https://www.idref.fr/147099072</idno>
                  <affiliation ref="#struct-117974"/>
                </author>
              </analytic>
              <monogr>
                <meeting>
                  <title>IJCNN 2020</title>
                  <date type="start">2020-07-19</date>
                  <settlement>Glasgow</settlement>
                  <country key="GB">United Kingdom</country>
                </meeting>
                <imprint/>
              </monogr>
            </biblStruct>
          </sourceDesc>
          <profileDesc>
            <langUsage>
              <language ident="en">English</language>
            </langUsage>
            <textClass>
              <keywords scheme="author">
                <term xml:lang="en">discrete information bottleneck</term>
                <term xml:lang="en">k-means</term>
                <term xml:lang="en">VQ-VAE</term>
              </keywords>
              <classCode scheme="halDomain" n="info.info-cl">Computer Science [cs]/Computation and Language [cs.CL]</classCode>
              <classCode scheme="halDomain" n="info.info-ai">Computer Science [cs]/Artificial Intelligence [cs.AI]</classCode>
              <classCode scheme="halDomain" n="info.info-ne">Computer Science [cs]/Neural and Evolutionary Computing [cs.NE]</classCode>
              <classCode scheme="halDomain" n="info.info-lg">Computer Science [cs]/Machine Learning [cs.LG]</classCode>
              <classCode scheme="halTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halOldTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halTreeTypology" n="COMM">Conference papers</classCode>
            </textClass>
            <abstract xml:lang="en">
              <p>In this paper we demonstrate methods for reliable and efficient training of discrete representation using Vector-Quantized Variational Auto-Encoder models (VQ-VAEs). Discrete latent variable models have been shown to learn nontrivial representations of speech, applicable to unsupervised voice conversion and reaching state-of-the-art performance on unit discovery tasks. For unsupervised representation learning, they became viable alternatives to continuous latent variable models such as the Variational Auto-Encoder (VAE). However, training deep discrete variable models is challenging, due to the inherent non-differentiability of the discretization operation. In this paper we focus on VQ-VAE, a state-of-the-art discrete bottleneck model shown to perform on par with its continuous counterparts. It quantizes encoder outputs with on-line k-means clustering. We show that the codebook learning can suffer from poor initialization and non-stationarity of clustered encoder outputs. We demonstrate that these can be successfully overcome by increasing the learning rate for the codebook and periodic date-dependent codeword re-initialization. As a result, we achieve more robust training across different tasks, and significantly increase the usage of latent codewords even for large code-books. This has practical benefit, for instance, in unsupervised representation learning, where large codebooks may lead to disentanglement of latent representations.</p>
            </abstract>
          </profileDesc>
        </biblFull>
      </listBibl>
    </body>
    <back>
      <listOrg type="structures">
        <org type="institution" xml:id="struct-305685" status="VALID">
          <idno type="IdRef">026572346</idno>
          <idno type="ISNI">0000000121853326</idno>
          <idno type="ROR">https://ror.org/00yae6e25</idno>
          <idno type="Wikidata">Q315658</idno>
          <orgName>University of Wrocław [Poland] = Uniwersytet Wrocławski [Polska] = Universität Breslau [Polen] = Université de Wrocław [Pologne]</orgName>
          <orgName type="acronym">UWr</orgName>
          <date type="start">1702-01-01</date>
          <desc>
            <address>
              <addrLine>Place Uniwersytecki 1, 50-137 Wrocław, Pologne</addrLine>
              <country key="PL"/>
            </address>
            <ref type="url">https://uwr.edu.pl</ref>
          </desc>
        </org>
        <org type="researchteam" xml:id="struct-530702" status="VALID">
          <orgName>DYNamiques de l’Information</orgName>
          <orgName type="acronym">DYNI</orgName>
          <desc>
            <address>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.lis-lab.fr/dyni</ref>
          </desc>
          <listRelation>
            <relation active="#struct-527033" type="direct"/>
            <relation name="UMR7020" active="#struct-198056" type="indirect"/>
            <relation name="UMR7020" active="#struct-303091" type="indirect"/>
            <relation name="UMR7020" active="#struct-441569" type="indirect"/>
          </listRelation>
        </org>
        <org type="laboratory" xml:id="struct-527033" status="VALID">
          <idno type="IdRef">234768002</idno>
          <idno type="RNSR">201822714R</idno>
          <idno type="ROR">https://ror.org/0257sgk90</idno>
          <idno type="Wikidata">Q52605252</idno>
          <orgName>Laboratoire d'Informatique et des Systèmes (LIS) (Marseille, Toulon)</orgName>
          <orgName type="acronym">LIS</orgName>
          <date type="start">2018-01-01</date>
          <desc>
            <address>
              <addrLine>Aix Marseille Université – Campus de Saint Jérôme – Bat. Polytech, 52 Av. Escadrille Normandie Niemen, 13397 Marseille Cedex 20</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.lis-lab.fr</ref>
          </desc>
          <listRelation>
            <relation name="UMR7020" active="#struct-198056" type="direct"/>
            <relation name="UMR7020" active="#struct-303091" type="direct"/>
            <relation name="UMR7020" active="#struct-441569" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-300813" status="VALID">
          <idno type="ROR">https://ror.org/04v3ywz14</idno>
          <orgName>China Agricultural University</orgName>
          <orgName type="acronym">CAU</orgName>
          <desc>
            <address>
              <addrLine>17 Tsinghua East Road, Beijing, 100083, P. R. China</addrLine>
              <country key="CN"/>
            </address>
            <ref type="url">http://en.cau.edu.cn/</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-310135" status="VALID">
          <idno type="IdRef">08335302X</idno>
          <idno type="ROR">https://ror.org/0443cwa12</idno>
          <orgName>Tallinn University of Technology</orgName>
          <orgName type="acronym">TalTech</orgName>
          <date type="start">1918-01-01</date>
          <desc>
            <address>
              <addrLine>Ehitajate tee 5, 19086 Tallinn</addrLine>
              <country key="EE"/>
            </address>
            <ref type="url">https://taltech.ee/en/</ref>
          </desc>
        </org>
        <org type="laboratory" xml:id="struct-117974" status="VALID">
          <idno type="IdRef">035240776</idno>
          <idno type="RNSR">200615275G</idno>
          <idno type="ROR">https://ror.org/03gvgnv89</idno>
          <idno type="Wikidata">Q51779666</idno>
          <orgName>Laboratoire d'Informatique de l'Université du Mans</orgName>
          <orgName type="acronym">LIUM</orgName>
          <desc>
            <address>
              <addrLine>Avenue Laennec 72085 Le Mans cedex 9</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://lium.univ-lemans.fr/</ref>
          </desc>
          <listRelation>
            <relation name="EA4023" active="#struct-7566" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-198056" status="VALID">
          <idno type="IdRef">15863621X</idno>
          <idno type="ISNI">0000 0001 2176 4817</idno>
          <idno type="ROR">https://ror.org/035xkbk20</idno>
          <orgName>Aix Marseille Université</orgName>
          <orgName type="acronym">AMU</orgName>
          <date type="start">2012-01-01</date>
          <desc>
            <address>
              <addrLine>Aix-Marseille UniversitéJardins du Pharo58 Boulevard Charles Livon13284 Marseille cedex 7</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.univ-amu.fr/</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-303091" status="VALID">
          <idno type="IdRef">031122337</idno>
          <idno type="ROR">https://ror.org/02m9kbe37</idno>
          <orgName>Université de Toulon</orgName>
          <orgName type="acronym">UTLN</orgName>
          <desc>
            <address>
              <addrLine>Avenue de l'Université - BP 20132 - 83957 La Garde Cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.univ-tln.fr/</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-441569" status="VALID">
          <idno type="IdRef">02636817X</idno>
          <idno type="ISNI">0000000122597504</idno>
          <idno type="ROR">https://ror.org/02feahw73</idno>
          <orgName>Centre National de la Recherche Scientifique</orgName>
          <orgName type="acronym">CNRS</orgName>
          <date type="start">1939-10-19</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.cnrs.fr/</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-7566" status="VALID">
          <idno type="IdRef">026404435</idno>
          <idno type="ROR">https://ror.org/01mtcc283</idno>
          <orgName>Le Mans Université</orgName>
          <orgName type="acronym">UM</orgName>
          <desc>
            <address>
              <addrLine>Avenue Olivier Messiaen - 72085 Le Mans cedex 9</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.univ-lemans.fr/</ref>
          </desc>
        </org>
      </listOrg>
    </back>
  </text>
</TEI>