<?xml version="1.0" encoding="utf-8"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hal="http://hal.archives-ouvertes.fr/" xmlns:gml="http://www.opengis.net/gml/3.3/" xmlns:gmlce="http://www.opengis.net/gml/3.3/ce" version="1.1" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 http://api.archives-ouvertes.fr/documents/aofr-sword.xsd">
  <teiHeader>
    <fileDesc>
      <titleStmt>
        <title>HAL TEI export of hal-01876506</title>
      </titleStmt>
      <publicationStmt>
        <distributor>CCSD</distributor>
        <availability status="restricted">
          <licence target="https://creativecommons.org/publicdomain/zero/1.0/">CC0 1.0 - Universal</licence>
        </availability>
        <date when="2026-05-19T17:08:23+02:00"/>
      </publicationStmt>
      <sourceDesc>
        <p part="N">HAL API Platform</p>
      </sourceDesc>
    </fileDesc>
  </teiHeader>
  <text>
    <body>
      <listBibl>
        <biblFull>
          <titleStmt>
            <title xml:lang="en">Increasing Argument Annotation Reproducibility by Using Inter-annotator Agreement to Improve Guidelines</title>
            <author role="aut">
              <persName>
                <forename type="first">Milagro</forename>
                <surname>Teruel</surname>
              </persName>
              <idno type="halauthorid">1183345-0</idno>
              <affiliation ref="#struct-38336"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Cristian</forename>
                <surname>Cardellino</surname>
              </persName>
              <email type="md5">efc01313e3e43b2fc06b4fc550eaaa90</email>
              <email type="domain">gmail.com</email>
              <ptr type="url" target="https://crscardellino.github.io"/>
              <idno type="idhal" notation="string">ccardellino</idno>
              <idno type="idhal" notation="numeric">1323423</idno>
              <idno type="halauthorid" notation="string">866814-1323423</idno>
              <idno type="GOOGLE SCHOLAR">YuIeGYEAAAAJ</idno>
              <idno type="ORCID">https://orcid.org/0009-0000-1129-8330</idno>
              <affiliation ref="#struct-38336"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Fernando</forename>
                <surname>Cardellino</surname>
              </persName>
              <idno type="halauthorid">1424037-0</idno>
              <affiliation ref="#struct-38336"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Laura</forename>
                <surname>Alonso Alemany</surname>
              </persName>
              <idno type="halauthorid">923310-0</idno>
              <affiliation ref="#struct-38336"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Serena</forename>
                <surname>Villata</surname>
              </persName>
              <email type="md5">344072e78dc9a8131294cd6155a40995</email>
              <email type="domain">i3s.unice.fr</email>
              <idno type="idhal" notation="string">serena-villata</idno>
              <idno type="idhal" notation="numeric">9409</idno>
              <idno type="halauthorid" notation="string">28004-9409</idno>
              <idno type="ORCID">https://orcid.org/0000-0003-3495-493X</idno>
              <idno type="IDREF">https://www.idref.fr/200242911</idno>
              <idno type="VIAF">https://viaf.org/viaf/28150202128003110141</idno>
              <idno type="ISNI">http://isni.org/isni/0000000498435124</idno>
              <idno type="GOOGLE SCHOLAR">https://scholar.google.fr/citations?user=Sy9G0pUAAAAJ</idno>
              <affiliation ref="#struct-178918"/>
              <affiliation ref="#struct-441569"/>
            </author>
            <editor role="depositor">
              <persName>
                <forename>Serena</forename>
                <surname>Villata</surname>
              </persName>
              <email type="md5">344072e78dc9a8131294cd6155a40995</email>
              <email type="domain">i3s.unice.fr</email>
            </editor>
          </titleStmt>
          <editionStmt>
            <edition n="v1" type="current">
              <date type="whenSubmitted">2018-09-18 14:54:24</date>
              <date type="whenModified">2026-01-13 08:20:03</date>
              <date type="whenReleased">2018-09-18 14:54:24</date>
              <date type="whenProduced">2018-05-07</date>
            </edition>
            <respStmt>
              <resp>contributor</resp>
              <name key="172450">
                <persName>
                  <forename>Serena</forename>
                  <surname>Villata</surname>
                </persName>
                <email type="md5">344072e78dc9a8131294cd6155a40995</email>
                <email type="domain">i3s.unice.fr</email>
              </name>
            </respStmt>
          </editionStmt>
          <publicationStmt>
            <distributor>CCSD</distributor>
            <idno type="halId">hal-01876506</idno>
            <idno type="halUri">https://hal.science/hal-01876506</idno>
            <idno type="halBibtex">teruel:hal-01876506</idno>
            <idno type="halRefHtml">&lt;i&gt;LREC 2018 - 11th International Conference on Language Resources and Evaluation&lt;/i&gt;, May 2018, Miyazaki, Japan. pp.1-4</idno>
            <idno type="halRef">LREC 2018 - 11th International Conference on Language Resources and Evaluation, May 2018, Miyazaki, Japan. pp.1-4</idno>
            <availability status="restricted"/>
          </publicationStmt>
          <seriesStmt>
            <idno type="stamp" n="UNICE">Université Nice Sophia Antipolis</idno>
            <idno type="stamp" n="CNRS">CNRS - Centre national de la recherche scientifique</idno>
            <idno type="stamp" n="INRIA">INRIA - Institut National de Recherche en Informatique et en Automatique</idno>
            <idno type="stamp" n="INRIA-SOPHIA">INRIA Sophia Antipolis - Méditerranée</idno>
            <idno type="stamp" n="I3S">Laboratoire d'Informatique, Signaux et Systèmes de Sophia-Antipolis</idno>
            <idno type="stamp" n="INRIASO">INRIA-SOPHIA</idno>
            <idno type="stamp" n="INRIA_TEST">INRIA - Institut National de Recherche en Informatique et en Automatique</idno>
            <idno type="stamp" n="TESTALAIN1">TESTALAIN1</idno>
            <idno type="stamp" n="WIMMICS">WIMMICS: Web-Instrumented huMan-Machine Interactions, Communities, and Semantics</idno>
            <idno type="stamp" n="INRIA2">INRIA 2</idno>
            <idno type="stamp" n="UNIV-COTEDAZUR">Université Côte d'Azur</idno>
            <idno type="stamp" n="TEST-HALCNRS">Collection test HAL CNRS</idno>
            <idno type="stamp" n="TEST-NICE">test nice</idno>
          </seriesStmt>
          <notesStmt>
            <note type="audience" n="2">International</note>
            <note type="invited" n="0">No</note>
            <note type="popular" n="0">No</note>
            <note type="peer" n="1">Yes</note>
            <note type="proceedings" n="0">No</note>
          </notesStmt>
          <sourceDesc>
            <biblStruct>
              <analytic>
                <title xml:lang="en">Increasing Argument Annotation Reproducibility by Using Inter-annotator Agreement to Improve Guidelines</title>
                <author role="aut">
                  <persName>
                    <forename type="first">Milagro</forename>
                    <surname>Teruel</surname>
                  </persName>
                  <idno type="halauthorid">1183345-0</idno>
                  <affiliation ref="#struct-38336"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Cristian</forename>
                    <surname>Cardellino</surname>
                  </persName>
                  <email type="md5">efc01313e3e43b2fc06b4fc550eaaa90</email>
                  <email type="domain">gmail.com</email>
                  <ptr type="url" target="https://crscardellino.github.io"/>
                  <idno type="idhal" notation="string">ccardellino</idno>
                  <idno type="idhal" notation="numeric">1323423</idno>
                  <idno type="halauthorid" notation="string">866814-1323423</idno>
                  <idno type="GOOGLE SCHOLAR">YuIeGYEAAAAJ</idno>
                  <idno type="ORCID">https://orcid.org/0009-0000-1129-8330</idno>
                  <affiliation ref="#struct-38336"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Fernando</forename>
                    <surname>Cardellino</surname>
                  </persName>
                  <idno type="halauthorid">1424037-0</idno>
                  <affiliation ref="#struct-38336"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Laura</forename>
                    <surname>Alonso Alemany</surname>
                  </persName>
                  <idno type="halauthorid">923310-0</idno>
                  <affiliation ref="#struct-38336"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Serena</forename>
                    <surname>Villata</surname>
                  </persName>
                  <email type="md5">344072e78dc9a8131294cd6155a40995</email>
                  <email type="domain">i3s.unice.fr</email>
                  <idno type="idhal" notation="string">serena-villata</idno>
                  <idno type="idhal" notation="numeric">9409</idno>
                  <idno type="halauthorid" notation="string">28004-9409</idno>
                  <idno type="ORCID">https://orcid.org/0000-0003-3495-493X</idno>
                  <idno type="IDREF">https://www.idref.fr/200242911</idno>
                  <idno type="VIAF">https://viaf.org/viaf/28150202128003110141</idno>
                  <idno type="ISNI">http://isni.org/isni/0000000498435124</idno>
                  <idno type="GOOGLE SCHOLAR">https://scholar.google.fr/citations?user=Sy9G0pUAAAAJ</idno>
                  <affiliation ref="#struct-178918"/>
                  <affiliation ref="#struct-441569"/>
                </author>
              </analytic>
              <monogr>
                <title level="m">Proceedings of the Eleventh International Conference on Language Resources and Evaluation, LREC 2018</title>
                <meeting>
                  <title>LREC 2018 - 11th International Conference on Language Resources and Evaluation</title>
                  <date type="start">2018-05-07</date>
                  <date type="end">2018-05-12</date>
                  <settlement>Miyazaki</settlement>
                  <country key="JP">Japan</country>
                </meeting>
                <imprint>
                  <biblScope unit="pp">1-4</biblScope>
                </imprint>
              </monogr>
            </biblStruct>
          </sourceDesc>
          <profileDesc>
            <langUsage>
              <language ident="en">English</language>
            </langUsage>
            <textClass>
              <keywords scheme="author">
                <term xml:lang="en">Argument Mining</term>
                <term xml:lang="en">Annotation Guidelines</term>
                <term xml:lang="en">Reproducibility</term>
              </keywords>
              <classCode scheme="halDomain" n="scco.comp">Cognitive science/Computer science</classCode>
              <classCode scheme="halDomain" n="scco.ling">Cognitive science/Linguistics</classCode>
              <classCode scheme="halTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halOldTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halTreeTypology" n="COMM">Conference papers</classCode>
            </textClass>
            <abstract xml:lang="en">
              <p>In this abstract we present a methodology to improve Argument annotation guidelines by exploiting inter-annotator agreement measures.After a first stage of the annotation effort, we have detected problematic issues via an analysis of inter-annotator agreement. Wehave detected ill-defined concepts, which we have addressed by redefining high-level annotation goals. For other concepts, thatare well-delimited but complex, the annotation protocol has been extended and detailed. Moreover, as can be expected, we showthat distinctions where human annotators have less agreement are also those where automatic analyzers perform worse. Thus, thereproducibility of results of Argument Mining systems can be addressed by improving inter-annotator agreement in the training material.Following this methodology, we are enhancing a corpus annotated with argumentation, available at https://github.com/PLN-FaMAF/ArgumentMiningECHR together with guidelines and analyses of agreement. These analyses can be used to filterperformance figures of automated systems, with lower penalties for cases where human annotators agree less.</p>
            </abstract>
          </profileDesc>
        </biblFull>
      </listBibl>
    </body>
    <back>
      <listOrg type="structures">
        <org type="institution" xml:id="struct-38336" status="VALID">
          <idno type="ROR">https://ror.org/056tb7j80</idno>
          <orgName>Universidad Nacional de Córdoba [Argentina]</orgName>
          <desc>
            <address>
              <addrLine>Av. Haya de la Torre s/n - Pabellón Argentina</addrLine>
              <country key="AR"/>
            </address>
            <ref type="url">http://www.unc.edu.ar</ref>
          </desc>
        </org>
        <org type="researchteam" xml:id="struct-178918" status="VALID">
          <idno type="RNSR">201221031M</idno>
          <idno type="ROR">https://ror.org/04hsaft65</idno>
          <orgName>Web-Instrumented huMan-Machine Interactions, Communities and Semantics</orgName>
          <orgName type="acronym">WIMMICS</orgName>
          <date type="start">2013-07-01</date>
          <date type="end">2029-04-30</date>
          <desc>
            <address>
              <addrLine>2004 route des lucioles06902 Sophia Antipolis</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://wimmics.inria.fr/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-34586" type="direct"/>
            <relation active="#struct-300009" type="indirect"/>
            <relation active="#struct-452156" type="direct"/>
            <relation active="#struct-13009" type="indirect"/>
            <relation active="#struct-117617" type="indirect"/>
            <relation name="UMR7271" active="#struct-441569" type="indirect"/>
            <relation active="#struct-1039632" type="indirect"/>
          </listRelation>
        </org>
        <org type="regroupinstitution" xml:id="struct-441569" status="VALID">
          <idno type="IdRef">02636817X</idno>
          <idno type="ISNI">0000000122597504</idno>
          <idno type="ROR">https://ror.org/02feahw73</idno>
          <orgName>Centre National de la Recherche Scientifique</orgName>
          <orgName type="acronym">CNRS</orgName>
          <date type="start">1939-10-19</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.cnrs.fr/</ref>
          </desc>
        </org>
        <org type="laboratory" xml:id="struct-34586" status="VALID">
          <idno type="RNSR">198318250R</idno>
          <idno type="ROR">https://ror.org/01nzkaw91</idno>
          <orgName>Centre Inria d'Université Côte d'Azur</orgName>
          <desc>
            <address>
              <addrLine>2004 route des Lucioles BP 93 06902 Sophia Antipolis</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/centre/sophia/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-300009" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-300009" status="VALID">
          <idno type="ROR">https://ror.org/02kvxyf05</idno>
          <orgName>Institut National de Recherche en Informatique et en Automatique</orgName>
          <orgName type="acronym">Inria</orgName>
          <desc>
            <address>
              <addrLine>Domaine de VoluceauRocquencourt - BP 10578153 Le Chesnay Cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/en/</ref>
          </desc>
        </org>
        <org type="department" xml:id="struct-452156" status="VALID">
          <orgName>Scalable and Pervasive softwARe and Knowledge Systems</orgName>
          <orgName type="acronym">Laboratoire I3S - SPARKS</orgName>
          <date type="start">2016-03-03</date>
          <desc>
            <address>
              <addrLine>Laboratoire I3SCS 4012106903 Sophia Antipolis Cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.i3s.unice.fr/sparks</ref>
          </desc>
          <listRelation>
            <relation active="#struct-13009" type="direct"/>
            <relation active="#struct-117617" type="indirect"/>
            <relation name="UMR7271" active="#struct-441569" type="indirect"/>
            <relation active="#struct-1039632" type="indirect"/>
          </listRelation>
        </org>
        <org type="laboratory" xml:id="struct-13009" status="VALID">
          <idno type="IdRef">050320491</idno>
          <idno type="ISNI">0000000105613840</idno>
          <idno type="RNSR">201220432L</idno>
          <idno type="ROR">https://ror.org/01215r597</idno>
          <orgName>Laboratoire d'Informatique, Signaux, et Systèmes de Sophia Antipolis</orgName>
          <orgName type="acronym">I3S</orgName>
          <desc>
            <address>
              <addrLine>2000, route des Lucioles - Les Algorithmes - bât. Euclide B 06900 Sophia Antipolis</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.i3s.unice.fr/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-117617" type="direct"/>
            <relation name="UMR7271" active="#struct-441569" type="direct"/>
            <relation active="#struct-1039632" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-117617" status="VALID">
          <idno type="IdRef">026403498</idno>
          <idno type="ISNI">0000000123372892</idno>
          <idno type="ROR">https://ror.org/02k9vew78</idno>
          <orgName>Université Nice Sophia Antipolis (1965 - 2019)</orgName>
          <orgName type="acronym">UNS</orgName>
          <date type="start">1965-10-23</date>
          <date type="end">2019-12-31</date>
          <desc>
            <address>
              <addrLine>Parc Valrose, 06100 Nice</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://unice.fr/</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-1039632" status="VALID">
          <idno type="IdRef">241035694</idno>
          <idno type="ROR">https://ror.org/019tgvf94</idno>
          <orgName>Université Côte d'Azur</orgName>
          <orgName type="acronym">UniCA</orgName>
          <date type="start">2020-01-01</date>
          <desc>
            <address>
              <addrLine>Parc Valrose, 28, avenue Valrose 06108 Nice Cedex 2</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://univ-cotedazur.fr</ref>
          </desc>
        </org>
      </listOrg>
    </back>
  </text>
</TEI>