<?xml version="1.0" encoding="utf-8"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hal="http://hal.archives-ouvertes.fr/" xmlns:gml="http://www.opengis.net/gml/3.3/" xmlns:gmlce="http://www.opengis.net/gml/3.3/ce" version="1.1" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 http://api.archives-ouvertes.fr/documents/aofr-sword.xsd">
  <teiHeader>
    <fileDesc>
      <titleStmt>
        <title>HAL TEI export of hal-03718886</title>
      </titleStmt>
      <publicationStmt>
        <distributor>CCSD</distributor>
        <availability status="restricted">
          <licence target="https://creativecommons.org/publicdomain/zero/1.0/">CC0 1.0 - Universal</licence>
        </availability>
        <date when="2026-05-20T00:13:09+02:00"/>
      </publicationStmt>
      <sourceDesc>
        <p part="N">HAL API Platform</p>
      </sourceDesc>
    </fileDesc>
  </teiHeader>
  <text>
    <body>
      <listBibl>
        <biblFull>
          <titleStmt>
            <title xml:lang="en">Analyzing the Impact of Undersampling on the Benchmarking and Configuration of Evolutionary Algorithms</title>
            <author role="aut">
              <persName>
                <forename type="first">Diederick</forename>
                <surname>Vermetten</surname>
              </persName>
              <email type="md5">1961439e96bb90351470ed05b3afad92</email>
              <email type="domain">liacs.leidenuniv.nl</email>
              <idno type="idhal" notation="numeric">1148405</idno>
              <idno type="halauthorid" notation="string">1615144-1148405</idno>
              <affiliation ref="#struct-135222"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Hao</forename>
                <surname>Wang</surname>
              </persName>
              <idno type="idhal" notation="numeric">762829</idno>
              <idno type="halauthorid" notation="string">473542-762829</idno>
              <idno type="IDREF">https://www.idref.fr/177547057</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-9819-7483</idno>
              <affiliation ref="#struct-135222"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Manuel</forename>
                <surname>López-Ibañez</surname>
              </persName>
              <email type="md5">aec5fd4734e25fdf95cf98f785f2df41</email>
              <email type="domain">uma.es</email>
              <idno type="idhal" notation="numeric">1098607</idno>
              <idno type="halauthorid" notation="string">2545133-1098607</idno>
              <affiliation ref="#struct-198404"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Carola</forename>
                <surname>Doerr</surname>
              </persName>
              <email type="md5">d1b2e33c2d3f25d09951b4c49fa97770</email>
              <email type="domain">lip6.fr</email>
              <idno type="idhal" notation="string">carola-doerr</idno>
              <idno type="idhal" notation="numeric">1290</idno>
              <idno type="halauthorid" notation="string">17976-1290</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-4981-3227</idno>
              <idno type="IDREF">https://www.idref.fr/232603383</idno>
              <affiliation ref="#struct-541718"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Thomas</forename>
                <surname>Bäck</surname>
              </persName>
              <idno type="halauthorid">1456083-0</idno>
            </author>
            <editor role="depositor">
              <persName>
                <forename>Carola</forename>
                <surname>Doerr</surname>
              </persName>
              <email type="md5">d1b2e33c2d3f25d09951b4c49fa97770</email>
              <email type="domain">lip6.fr</email>
            </editor>
          </titleStmt>
          <editionStmt>
            <edition n="v1" type="current">
              <date type="whenSubmitted">2022-07-13 09:53:09</date>
              <date type="whenWritten">2022</date>
              <date type="whenModified">2024-10-30 13:34:17</date>
              <date type="whenReleased">2022-07-13 10:05:06</date>
              <date type="whenProduced">2022-07-09</date>
              <date type="whenEndEmbargoed">2022-07-13</date>
              <ref type="file" target="https://hal.science/hal-03718886v1/document">
                <date notBefore="2022-07-13"/>
              </ref>
              <ref type="file" subtype="author" n="1" target="https://hal.science/hal-03718886v1/file/GECCO-variability-2204.09353.pdf" id="file-3718886-3251154">
                <date notBefore="2022-07-13"/>
              </ref>
            </edition>
            <respStmt>
              <resp>contributor</resp>
              <name key="303028">
                <persName>
                  <forename>Carola</forename>
                  <surname>Doerr</surname>
                </persName>
                <email type="md5">d1b2e33c2d3f25d09951b4c49fa97770</email>
                <email type="domain">lip6.fr</email>
              </name>
            </respStmt>
          </editionStmt>
          <publicationStmt>
            <distributor>CCSD</distributor>
            <idno type="halId">hal-03718886</idno>
            <idno type="halUri">https://hal.science/hal-03718886</idno>
            <idno type="halBibtex">vermetten:hal-03718886</idno>
            <idno type="halRefHtml">&lt;i&gt;GECCO '22: Genetic and Evolutionary Computation Conference&lt;/i&gt;, Jul 2022, Boston, United States. &lt;a target="_blank" href="https://dx.doi.org/10.1145/3512290.3528799"&gt;&amp;#x27E8;10.1145/3512290.3528799&amp;#x27E9;&lt;/a&gt;</idno>
            <idno type="halRef">GECCO '22: Genetic and Evolutionary Computation Conference, Jul 2022, Boston, United States. &amp;#x27E8;10.1145/3512290.3528799&amp;#x27E9;</idno>
            <availability status="restricted">
              <licence target="https://about.hal.science/hal-authorisation-v1/">HAL Authorization<ref corresp="#file-3718886-3251154"/></licence>
            </availability>
          </publicationStmt>
          <seriesStmt>
            <idno type="stamp" n="CNRS">CNRS - Centre national de la recherche scientifique</idno>
            <idno type="stamp" n="LIP6" corresp="SORBONNE-UNIVERSITE">Laboratoire d'Informatique de Paris 6</idno>
            <idno type="stamp" n="SORBONNE-UNIVERSITE">Sorbonne Université</idno>
            <idno type="stamp" n="SORBONNE-UNIV" corresp="SORBONNE-UNIVERSITE">Sorbonne Université 01/01/2018</idno>
            <idno type="stamp" n="SU-SCIENCES" corresp="SORBONNE-UNIVERSITE">Faculté des Sciences de Sorbonne Université</idno>
            <idno type="stamp" n="SU-TI">Sorbonne Université - Texte Intégral</idno>
            <idno type="stamp" n="ALLIANCE-SU"> Alliance Sorbonne Université</idno>
            <idno type="stamp" n="SUPRA_MATHS_INFO">Mathématiques + Informatique</idno>
          </seriesStmt>
          <notesStmt>
            <note type="audience" n="2">International</note>
            <note type="invited" n="0">No</note>
            <note type="popular" n="0">No</note>
            <note type="peer" n="1">Yes</note>
            <note type="proceedings" n="1">Yes</note>
          </notesStmt>
          <sourceDesc>
            <biblStruct>
              <analytic>
                <title xml:lang="en">Analyzing the Impact of Undersampling on the Benchmarking and Configuration of Evolutionary Algorithms</title>
                <author role="aut">
                  <persName>
                    <forename type="first">Diederick</forename>
                    <surname>Vermetten</surname>
                  </persName>
                  <email type="md5">1961439e96bb90351470ed05b3afad92</email>
                  <email type="domain">liacs.leidenuniv.nl</email>
                  <idno type="idhal" notation="numeric">1148405</idno>
                  <idno type="halauthorid" notation="string">1615144-1148405</idno>
                  <affiliation ref="#struct-135222"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Hao</forename>
                    <surname>Wang</surname>
                  </persName>
                  <idno type="idhal" notation="numeric">762829</idno>
                  <idno type="halauthorid" notation="string">473542-762829</idno>
                  <idno type="IDREF">https://www.idref.fr/177547057</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-9819-7483</idno>
                  <affiliation ref="#struct-135222"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Manuel</forename>
                    <surname>López-Ibañez</surname>
                  </persName>
                  <email type="md5">aec5fd4734e25fdf95cf98f785f2df41</email>
                  <email type="domain">uma.es</email>
                  <idno type="idhal" notation="numeric">1098607</idno>
                  <idno type="halauthorid" notation="string">2545133-1098607</idno>
                  <affiliation ref="#struct-198404"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Carola</forename>
                    <surname>Doerr</surname>
                  </persName>
                  <email type="md5">d1b2e33c2d3f25d09951b4c49fa97770</email>
                  <email type="domain">lip6.fr</email>
                  <idno type="idhal" notation="string">carola-doerr</idno>
                  <idno type="idhal" notation="numeric">1290</idno>
                  <idno type="halauthorid" notation="string">17976-1290</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-4981-3227</idno>
                  <idno type="IDREF">https://www.idref.fr/232603383</idno>
                  <affiliation ref="#struct-541718"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Thomas</forename>
                    <surname>Bäck</surname>
                  </persName>
                  <idno type="halauthorid">1456083-0</idno>
                </author>
              </analytic>
              <monogr>
                <title level="m">GECCO '22: Proceedings of the Genetic and Evolutionary Computation Conference</title>
                <meeting>
                  <title>GECCO '22: Genetic and Evolutionary Computation Conference</title>
                  <date type="start">2022-07-09</date>
                  <date type="end">2022-07-13</date>
                  <settlement>Boston</settlement>
                  <country key="US">United States</country>
                </meeting>
                <imprint>
                  <date type="datePub">2022</date>
                </imprint>
              </monogr>
              <idno type="doi">10.1145/3512290.3528799</idno>
            </biblStruct>
          </sourceDesc>
          <profileDesc>
            <langUsage>
              <language ident="en">English</language>
            </langUsage>
            <textClass>
              <keywords scheme="author">
                <term xml:lang="en">Parameter tuning</term>
                <term xml:lang="en">algorithm configuration</term>
                <term xml:lang="en">performance measures</term>
                <term xml:lang="en">evolution strategies</term>
              </keywords>
              <classCode scheme="halDomain" n="info.info-ne">Computer Science [cs]/Neural and Evolutionary Computing [cs.NE]</classCode>
              <classCode scheme="halTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halOldTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halTreeTypology" n="COMM">Conference papers</classCode>
            </textClass>
            <abstract xml:lang="en">
              <p>The stochastic nature of iterative optimization heuristics leads to inherently noisy performance measurements. Since these measurements are often gathered once and then used repeatedly, the number of collected samples will have a significant impact on the reliability of algorithm comparisons. We show that care should be taken when making decisions based on limited data. Particularly, we show that the number of runs used in many benchmarking studies, e.g., the default value of 15 suggested by the COCO environment, can be insufficient to reliably rank algorithms on well-known numerical optimization benchmarks. Additionally, methods for automated algorithm configuration are sensitive to insufficient sample sizes. This may result in the configurator choosing a "lucky" but poor-performing configuration despite exploring better ones. We show that relying on mean performance values, as many configurators do, can require a large number of runs to provide accurate comparisons between the considered configurations. Common statistical tests can greatly improve the situation in most cases but not always. We show examples of performance losses of more than 20%, even when using statistical races to dynamically adjust the number of runs, as done by irace. Our results underline the importance of appropriately considering the statistical distribution of performance values.</p>
            </abstract>
          </profileDesc>
        </biblFull>
      </listBibl>
    </body>
    <back>
      <listOrg type="structures">
        <org type="laboratory" xml:id="struct-135222" status="VALID">
          <orgName>Leiden Institute of Advanced Computer Science [Leiden]</orgName>
          <orgName type="acronym">LIACS</orgName>
          <desc>
            <address>
              <addrLine>Niels Bohrweg 1 2333 CA Leiden</addrLine>
              <country key="NL"/>
            </address>
            <ref type="url">http://www.liacs.nl/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-121723" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-198404" status="VALID">
          <orgName>Universidad de Málaga [Málaga] = University of Málaga [Málaga]</orgName>
          <desc>
            <address>
              <addrLine>Avda. Cervantes, 2   29071 MÁLAGA</addrLine>
              <country key="ES"/>
            </address>
            <ref type="url">http://www.uma.es/</ref>
          </desc>
        </org>
        <org type="researchteam" xml:id="struct-541718" status="VALID">
          <orgName>Recherche Opérationnelle</orgName>
          <orgName type="acronym">RO</orgName>
          <date type="start">2018-01-01</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
          </desc>
          <listRelation>
            <relation active="#struct-541703" type="direct"/>
            <relation active="#struct-413221" type="indirect"/>
            <relation name="UMR7606" active="#struct-441569" type="indirect"/>
          </listRelation>
        </org>
        <org type="regroupinstitution" xml:id="struct-121723" status="VALID">
          <idno type="ROR">https://ror.org/027bh9e22</idno>
          <orgName>Universiteit Leiden = Leiden University</orgName>
          <desc>
            <address>
              <addrLine>Leiden University | 2300 RA Leiden The Netherlands</addrLine>
              <country key="NL"/>
            </address>
            <ref type="url">https://www.universiteitleiden.nl/en/</ref>
          </desc>
        </org>
        <org type="laboratory" xml:id="struct-541703" status="VALID">
          <idno type="IdRef">13558292X</idno>
          <idno type="RNSR">199712651U</idno>
          <idno type="ROR">https://ror.org/05krcen59</idno>
          <orgName>LIP6</orgName>
          <date type="start">2018-01-01</date>
          <desc>
            <address>
              <addrLine>4 Place JUSSIEU 75252 PARIS CEDEX 05</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.lip6.fr/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-413221" type="direct"/>
            <relation name="UMR7606" active="#struct-441569" type="direct"/>
          </listRelation>
        </org>
        <org type="regroupinstitution" xml:id="struct-413221" status="VALID">
          <idno type="IdRef">221333754</idno>
          <idno type="ROR">https://ror.org/02en5vm52</idno>
          <orgName>Sorbonne Université</orgName>
          <orgName type="acronym">SU</orgName>
          <date type="start">2018-01-01</date>
          <desc>
            <address>
              <addrLine>21 rue de l’École de médecine - 75006 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.sorbonne-universite.fr/</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-441569" status="VALID">
          <idno type="IdRef">02636817X</idno>
          <idno type="ISNI">0000000122597504</idno>
          <idno type="ROR">https://ror.org/02feahw73</idno>
          <orgName>Centre National de la Recherche Scientifique</orgName>
          <orgName type="acronym">CNRS</orgName>
          <date type="start">1939-10-19</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.cnrs.fr/</ref>
          </desc>
        </org>
      </listOrg>
    </back>
  </text>
</TEI>