<?xml version="1.0" encoding="utf-8"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hal="http://hal.archives-ouvertes.fr/" xmlns:gml="http://www.opengis.net/gml/3.3/" xmlns:gmlce="http://www.opengis.net/gml/3.3/ce" version="1.1" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 http://api.archives-ouvertes.fr/documents/aofr-sword.xsd">
  <teiHeader>
    <fileDesc>
      <titleStmt>
        <title>HAL TEI export of hal-04132007</title>
      </titleStmt>
      <publicationStmt>
        <distributor>CCSD</distributor>
        <availability status="restricted">
          <licence target="https://creativecommons.org/publicdomain/zero/1.0/">CC0 1.0 - Universal</licence>
        </availability>
        <date when="2026-05-03T09:06:12+02:00"/>
      </publicationStmt>
      <sourceDesc>
        <p part="N">HAL API Platform</p>
      </sourceDesc>
    </fileDesc>
  </teiHeader>
  <text>
    <body>
      <listBibl>
        <biblFull>
          <titleStmt>
            <title xml:lang="en">Recovering quality scores in noisy pairwise subjective experiments using Negative log-likelihood</title>
            <author role="aut">
              <persName>
                <forename type="first">Andréas</forename>
                <surname>Pastor</surname>
              </persName>
              <email type="md5">b45a6e34afcedc4a510a289c97206cf4</email>
              <email type="domain">univ-nantes.fr</email>
              <idno type="idhal" notation="string">andreas-pastor</idno>
              <idno type="idhal" notation="numeric">750227</idno>
              <idno type="halauthorid" notation="string">2638765-750227</idno>
              <idno type="GOOGLE SCHOLAR">https://scholar.google.com/citations?user=IzzRZO8AAAAJ&amp;hl=fr</idno>
              <affiliation ref="#struct-1088573"/>
              <affiliation ref="#struct-1088564"/>
              <affiliation ref="#struct-1083366"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Lukáš</forename>
                <surname>Krasula</surname>
              </persName>
              <idno type="halauthorid">1079483-0</idno>
              <affiliation ref="#struct-550783"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Xiaoqing</forename>
                <surname>Zhu</surname>
              </persName>
              <idno type="halauthorid">943112-0</idno>
              <affiliation ref="#struct-550783"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Zhi</forename>
                <surname>Li</surname>
              </persName>
              <email type="md5">bf45511139f2e27f3785fadb722f424b</email>
              <email type="domain">netflix.com</email>
              <idno type="idhal" notation="numeric">1262816</idno>
              <idno type="halauthorid" notation="string">30175-1262816</idno>
              <affiliation ref="#struct-550783"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Patrick Le</forename>
                <surname>Callet</surname>
              </persName>
              <email type="md5">74e517cfff9225f3fd9416965f3fd87c</email>
              <email type="domain">univ-nantes.fr</email>
              <idno type="idhal" notation="string">patrick-le-callet</idno>
              <idno type="idhal" notation="numeric">15969</idno>
              <idno type="halauthorid" notation="string">1260953-15969</idno>
              <idno type="GOOGLE SCHOLAR">https://scholar.google.fr/citations?user=llgwlUgAAAAJ&amp;hl=en</idno>
              <idno type="IDREF">https://www.idref.fr/060370068</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-2143-7063</idno>
              <idno type="VIAF">https://viaf.org/viaf/173368181</idno>
              <affiliation ref="#struct-1088573"/>
              <affiliation ref="#struct-1088564"/>
              <affiliation ref="#struct-1083366"/>
              <affiliation ref="#struct-56663"/>
            </author>
            <editor role="depositor">
              <persName>
                <forename>Andréas</forename>
                <surname>Pastor</surname>
              </persName>
              <email type="md5">b45a6e34afcedc4a510a289c97206cf4</email>
              <email type="domain">univ-nantes.fr</email>
            </editor>
          </titleStmt>
          <editionStmt>
            <edition n="v1" type="current">
              <date type="whenSubmitted">2023-06-17 21:15:58</date>
              <date type="whenModified">2026-02-07 05:25:33</date>
              <date type="whenReleased">2023-06-19 10:01:50</date>
              <date type="whenProduced">2023-10-08</date>
              <date type="whenEndEmbargoed">2023-06-17</date>
              <ref type="file" target="https://hal.science/hal-04132007v1/document">
                <date notBefore="2023-06-17"/>
              </ref>
              <ref type="file" subtype="author" n="1" target="https://hal.science/hal-04132007v1/file/ICIP2023_spammers_preference%20%281%29.pdf" id="file-4132007-3597217">
                <date notBefore="2023-06-17"/>
              </ref>
            </edition>
            <respStmt>
              <resp>contributor</resp>
              <name key="1100720">
                <persName>
                  <forename>Andréas</forename>
                  <surname>Pastor</surname>
                </persName>
                <email type="md5">b45a6e34afcedc4a510a289c97206cf4</email>
                <email type="domain">univ-nantes.fr</email>
              </name>
            </respStmt>
          </editionStmt>
          <publicationStmt>
            <distributor>CCSD</distributor>
            <idno type="halId">hal-04132007</idno>
            <idno type="halUri">https://hal.science/hal-04132007</idno>
            <idno type="halBibtex">pastor:hal-04132007</idno>
            <idno type="halRefHtml">&lt;i&gt;2023 IEEE International Conference on Image Processing (ICIP)&lt;/i&gt;, Oct 2023, Kuala Lumpur, France. pp.2635-2639, &lt;a target="_blank" href="https://dx.doi.org/10.1109/ICIP49359.2023.10222071"&gt;&amp;#x27E8;10.1109/ICIP49359.2023.10222071&amp;#x27E9;&lt;/a&gt;</idno>
            <idno type="halRef">2023 IEEE International Conference on Image Processing (ICIP), Oct 2023, Kuala Lumpur, France. pp.2635-2639, &amp;#x27E8;10.1109/ICIP49359.2023.10222071&amp;#x27E9;</idno>
            <availability status="restricted">
              <licence target="https://about.hal.science/hal-authorisation-v1/">HAL Authorization<ref corresp="#file-4132007-3597217"/></licence>
            </availability>
          </publicationStmt>
          <seriesStmt>
            <idno type="stamp" n="CNRS">CNRS - Centre national de la recherche scientifique</idno>
            <idno type="stamp" n="EC-NANTES">Ecole Centrale de Nantes</idno>
            <idno type="stamp" n="UNAM">l'unam - université nantes angers le mans</idno>
            <idno type="stamp" n="LS2N">Laboratoire des Sciences du Numérique de Nantes</idno>
            <idno type="stamp" n="LS2N-IPI" corresp="LS2N">LS2N - équipe IPI ( Image Perception Interaction )</idno>
            <idno type="stamp" n="INSTITUTS-TELECOM">composantes instituts telecom </idno>
            <idno type="stamp" n="NANTES-UNIVERSITE">Nantes Université</idno>
            <idno type="stamp" n="NANTES-UNIV">Nantes Université</idno>
          </seriesStmt>
          <notesStmt>
            <note type="audience" n="2">International</note>
            <note type="invited" n="0">No</note>
            <note type="popular" n="0">No</note>
            <note type="peer" n="1">Yes</note>
            <note type="proceedings" n="0">No</note>
          </notesStmt>
          <sourceDesc>
            <biblStruct>
              <analytic>
                <title xml:lang="en">Recovering quality scores in noisy pairwise subjective experiments using Negative log-likelihood</title>
                <author role="aut">
                  <persName>
                    <forename type="first">Andréas</forename>
                    <surname>Pastor</surname>
                  </persName>
                  <email type="md5">b45a6e34afcedc4a510a289c97206cf4</email>
                  <email type="domain">univ-nantes.fr</email>
                  <idno type="idhal" notation="string">andreas-pastor</idno>
                  <idno type="idhal" notation="numeric">750227</idno>
                  <idno type="halauthorid" notation="string">2638765-750227</idno>
                  <idno type="GOOGLE SCHOLAR">https://scholar.google.com/citations?user=IzzRZO8AAAAJ&amp;hl=fr</idno>
                  <affiliation ref="#struct-1088573"/>
                  <affiliation ref="#struct-1088564"/>
                  <affiliation ref="#struct-1083366"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Lukáš</forename>
                    <surname>Krasula</surname>
                  </persName>
                  <idno type="halauthorid">1079483-0</idno>
                  <affiliation ref="#struct-550783"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Xiaoqing</forename>
                    <surname>Zhu</surname>
                  </persName>
                  <idno type="halauthorid">943112-0</idno>
                  <affiliation ref="#struct-550783"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Zhi</forename>
                    <surname>Li</surname>
                  </persName>
                  <email type="md5">bf45511139f2e27f3785fadb722f424b</email>
                  <email type="domain">netflix.com</email>
                  <idno type="idhal" notation="numeric">1262816</idno>
                  <idno type="halauthorid" notation="string">30175-1262816</idno>
                  <affiliation ref="#struct-550783"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Patrick Le</forename>
                    <surname>Callet</surname>
                  </persName>
                  <email type="md5">74e517cfff9225f3fd9416965f3fd87c</email>
                  <email type="domain">univ-nantes.fr</email>
                  <idno type="idhal" notation="string">patrick-le-callet</idno>
                  <idno type="idhal" notation="numeric">15969</idno>
                  <idno type="halauthorid" notation="string">1260953-15969</idno>
                  <idno type="GOOGLE SCHOLAR">https://scholar.google.fr/citations?user=llgwlUgAAAAJ&amp;hl=en</idno>
                  <idno type="IDREF">https://www.idref.fr/060370068</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-2143-7063</idno>
                  <idno type="VIAF">https://viaf.org/viaf/173368181</idno>
                  <affiliation ref="#struct-1088573"/>
                  <affiliation ref="#struct-1088564"/>
                  <affiliation ref="#struct-1083366"/>
                  <affiliation ref="#struct-56663"/>
                </author>
              </analytic>
              <monogr>
                <meeting>
                  <title>2023 IEEE International Conference on Image Processing (ICIP)</title>
                  <date type="start">2023-10-08</date>
                  <date type="end">2023-10-11</date>
                  <settlement>Kuala Lumpur</settlement>
                  <country key="FR">France</country>
                </meeting>
                <imprint>
                  <publisher>IEEE</publisher>
                  <biblScope unit="pp">2635-2639</biblScope>
                </imprint>
              </monogr>
              <idno type="doi">10.1109/ICIP49359.2023.10222071</idno>
            </biblStruct>
          </sourceDesc>
          <profileDesc>
            <langUsage>
              <language ident="en">English</language>
            </langUsage>
            <textClass>
              <keywords scheme="author">
                <term xml:lang="en">active-sampling</term>
                <term xml:lang="en">pair comparison</term>
                <term xml:lang="en">crowdsourcing</term>
                <term xml:lang="en">Spammer removal</term>
              </keywords>
              <classCode scheme="halDomain" n="info">Computer Science [cs]</classCode>
              <classCode scheme="halTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halOldTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halTreeTypology" n="COMM">Conference papers</classCode>
            </textClass>
            <abstract xml:lang="en">
              <p>To gather larger datasets to train data-angry deep learning quality assessment models, crowdsourcing has become essential to recruit participants. These participants are asked their opinion by directly rating stimuli, e.g., using single or double stimulus methodologies, or indirectly by ranking stimuli or comparing distances as in the Maximum Likelihood Difference Scaling method. In crowdsourcing, participants' behaviors and environmental distractions are not controlled. So, the researcher must pay attention to the answers' reliability. Cleaning methods exist for direct annotation subjective methodologies. However, solutions for indirect annotation methods are limited. In this work, we propose a method based on the negative log-likelihood to detect spammers among participants from their answers. To demonstrate its use, we applied it in a quadruplet preference-based scenario. The proposed method requires low computation and can be integrated into active-sampling strategies, where annotations available per comparison are small. We demonstrate that our method is robust to various spammer behaviors and accurate by removing only spammers. It helps reduce the gap between data collected in in-lab conditions (i.e., no spammer) and through crowdsourcing: our method reduces estimated uncertainties around data-points by 50%, and RMSE between estimations from an in-lab experiment and the same experiment performed in crowdsourcing by 1.8.</p>
            </abstract>
          </profileDesc>
        </biblFull>
      </listBibl>
    </body>
    <back>
      <listOrg type="structures">
        <org type="researchteam" xml:id="struct-1088573" status="VALID">
          <orgName>Image Perception Interaction</orgName>
          <orgName type="acronym">LS2N - équipe IPI</orgName>
          <date type="start">2022-01-01</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
          </desc>
          <listRelation>
            <relation active="#struct-1088564" type="direct"/>
            <relation active="#struct-300009" type="indirect"/>
            <relation name="UMR6004" active="#struct-441569" type="indirect"/>
            <relation active="#struct-481355" type="indirect"/>
            <relation active="#struct-302102" type="indirect"/>
            <relation active="#struct-1086798" type="indirect"/>
            <relation active="#struct-1083366" type="indirect"/>
            <relation active="#struct-1088543" type="indirect"/>
            <relation active="#struct-1083400" type="indirect"/>
          </listRelation>
        </org>
        <org type="laboratory" xml:id="struct-1088564" status="VALID">
          <idno type="IdRef">203637585</idno>
          <idno type="RNSR">201722241F</idno>
          <idno type="ROR">https://ror.org/02snf8m58</idno>
          <orgName>Laboratoire des Sciences du Numérique de Nantes</orgName>
          <orgName type="acronym">LS2N</orgName>
          <date type="start">2022-01-01</date>
          <desc>
            <address>
              <addrLine>Nantes Université – faculté des Sciences et Techniques (FST)2 Chemin de la HoussinièreBP 92208, 44322 Nantes Cedex 3</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.ls2n.fr</ref>
          </desc>
          <listRelation>
            <relation active="#struct-300009" type="direct"/>
            <relation name="UMR6004" active="#struct-441569" type="direct"/>
            <relation active="#struct-481355" type="direct"/>
            <relation active="#struct-302102" type="indirect"/>
            <relation active="#struct-1086798" type="direct"/>
            <relation active="#struct-1083366" type="indirect"/>
            <relation active="#struct-1088543" type="direct"/>
            <relation active="#struct-1083400" type="indirect"/>
          </listRelation>
        </org>
        <org type="regroupinstitution" xml:id="struct-1083366" status="VALID">
          <idno type="IdRef">258086599</idno>
          <idno type="ROR">https://ror.org/03gnr7b55</idno>
          <orgName>Nantes Université</orgName>
          <orgName type="acronym">Nantes Univ</orgName>
          <date type="start">2022-01-01</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
          </desc>
        </org>
        <org type="institution" xml:id="struct-550783" status="INCOMING">
          <orgName>Netflix</orgName>
        </org>
        <org type="laboratory" xml:id="struct-56663" status="VALID">
          <idno type="IdRef">03442945X</idno>
          <idno type="ISNI">0000000119314817</idno>
          <idno type="ROR">https://ror.org/055khg266</idno>
          <idno type="Wikidata">Q1665127</idno>
          <orgName>Institut universitaire de France</orgName>
          <orgName type="acronym">IUF</orgName>
          <desc>
            <address>
              <addrLine>Maison des Universités 103 Boulevard Saint-Michel 75005 Paris</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://iuf.amue.fr/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-301855" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-300009" status="VALID">
          <idno type="ROR">https://ror.org/02kvxyf05</idno>
          <orgName>Institut National de Recherche en Informatique et en Automatique</orgName>
          <orgName type="acronym">Inria</orgName>
          <desc>
            <address>
              <addrLine>Domaine de VoluceauRocquencourt - BP 10578153 Le Chesnay Cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/en/</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-441569" status="VALID">
          <idno type="IdRef">02636817X</idno>
          <idno type="ISNI">0000000122597504</idno>
          <idno type="ROR">https://ror.org/02feahw73</idno>
          <orgName>Centre National de la Recherche Scientifique</orgName>
          <orgName type="acronym">CNRS</orgName>
          <date type="start">1939-10-19</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.cnrs.fr/</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-481355" status="VALID">
          <idno type="IdRef">202743233</idno>
          <idno type="ROR">https://ror.org/030hj3061</idno>
          <orgName>IMT Atlantique</orgName>
          <orgName type="acronym">IMT Atlantique</orgName>
          <date type="start">2017-01-01</date>
          <desc>
            <address>
              <addrLine>Campus Brest : Technopôle Brest-Iroise CS 8381829238 BREST Cedex 3 -Campus Nantes : 4, rue Alfred Kastler- La chantrerie 44300 NANTES -Campus Rennes :  2 Rue de la Châtaigneraie, 35510 CESSON SEVIGNE</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.imt-atlantique.fr</ref>
          </desc>
          <listRelation>
            <relation active="#struct-302102" type="direct"/>
          </listRelation>
        </org>
        <org type="regroupinstitution" xml:id="struct-302102" status="VALID">
          <idno type="IdRef">192427156</idno>
          <idno type="ISNI">000000012202567X</idno>
          <idno type="ROR">https://ror.org/025vp2923</idno>
          <idno type="Wikidata">Q27962533</idno>
          <orgName>Institut Mines-Télécom [Paris]</orgName>
          <orgName type="acronym">IMT</orgName>
          <date type="start">2012-03-01</date>
          <desc>
            <address>
              <addrLine>19 Place Marguerite Perey, 91120 Palaiseau</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.imt.fr/</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-1086798" status="VALID">
          <idno type="IdRef">03063525X</idno>
          <idno type="ISNI">0000000122039289</idno>
          <idno type="ROR">https://ror.org/03nh7d505</idno>
          <orgName>NANTES UNIVERSITÉ - École Centrale de Nantes</orgName>
          <orgName type="acronym">Nantes Univ - ECN</orgName>
          <date type="start">2022-01-01</date>
          <desc>
            <address>
              <addrLine>1 rue de la Noë - BP 92101 - 44321 Nantes cedex 3</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.ec-nantes.fr/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-1083366" type="direct"/>
          </listRelation>
        </org>
        <org type="regrouplaboratory" xml:id="struct-1088543" status="VALID">
          <orgName>Nantes université - UFR des Sciences et des Techniques</orgName>
          <orgName type="acronym">Nantes univ -  UFR ST</orgName>
          <date type="start">2022-01-01</date>
          <desc>
            <address>
              <addrLine>2, rue de la Houssinière - BP 92208 - 44322 Nantes cedex 3</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.sciences-techniques.univ-nantes.fr/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-1083400" type="direct"/>
            <relation active="#struct-1083366" type="indirect"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-1083400" status="VALID">
          <orgName>Nantes Université - pôle Sciences et technologie</orgName>
          <date type="start">2022-01-01</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
            <ref type="url">https://nouvelleuniversite.univ-nantes.fr/les-poles</ref>
          </desc>
          <listRelation>
            <relation active="#struct-1083366" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-301855" status="VALID">
          <orgName>Ministère de l'Education nationale, de l’Enseignement supérieur et de la Recherche</orgName>
          <orgName type="acronym">M.E.N.E.S.R.</orgName>
          <desc>
            <address>
              <addrLine>1 rue Descartes - 75231 Paris cedex 05</addrLine>
              <country key="FR"/>
            </address>
          </desc>
        </org>
      </listOrg>
    </back>
  </text>
</TEI>