<?xml version="1.0" encoding="utf-8"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hal="http://hal.archives-ouvertes.fr/" xmlns:gml="http://www.opengis.net/gml/3.3/" xmlns:gmlce="http://www.opengis.net/gml/3.3/ce" version="1.1" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 http://api.archives-ouvertes.fr/documents/aofr-sword.xsd">
  <teiHeader>
    <fileDesc>
      <titleStmt>
        <title>HAL TEI export of hal-05230379</title>
      </titleStmt>
      <publicationStmt>
        <distributor>CCSD</distributor>
        <availability status="restricted">
          <licence target="https://creativecommons.org/publicdomain/zero/1.0/">CC0 1.0 - Universal</licence>
        </availability>
        <date when="2026-05-15T16:05:14+02:00"/>
      </publicationStmt>
      <sourceDesc>
        <p part="N">HAL API Platform</p>
      </sourceDesc>
    </fileDesc>
  </teiHeader>
  <text>
    <body>
      <listBibl>
        <biblFull>
          <titleStmt>
            <title xml:lang="en">Usefulness of LLMs as an Author Checklist Assistant for Scientific Papers: NeurIPS’24 Experiment</title>
            <author role="aut">
              <persName>
                <forename type="first">Alexander</forename>
                <surname>Goldberg</surname>
              </persName>
              <idno type="halauthorid">1409068-0</idno>
              <affiliation ref="#struct-67135"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Ihsan</forename>
                <surname>Ullah</surname>
              </persName>
              <idno type="halauthorid">551915-0</idno>
              <affiliation ref="#struct-410475"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Thanh Gia Hieu</forename>
                <surname>Khuong</surname>
              </persName>
              <email type="md5">4af4deff7f0f1e6756feed9d5f45d985</email>
              <email type="domain">gmail.com</email>
              <idno type="idhal" notation="string">ktgiahieu</idno>
              <idno type="idhal" notation="numeric">1282340</idno>
              <idno type="halauthorid" notation="string">2895062-1282340</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-2705-854X</idno>
              <affiliation ref="#struct-1066529"/>
              <affiliation ref="#struct-1061259"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Benedictus Kent</forename>
                <surname>Rachmat</surname>
              </persName>
              <email type="md5">444650a22d71d5ad3aaadb51fda9ff0b</email>
              <email type="domain">gmail.com</email>
              <idno type="idhal" notation="string">benedictus-kent-rachmat</idno>
              <idno type="idhal" notation="numeric">1282341</idno>
              <idno type="halauthorid" notation="string">2895063-1282341</idno>
              <idno type="ORCID">https://orcid.org/0009-0005-9200-7510</idno>
              <affiliation ref="#struct-1066529"/>
              <affiliation ref="#struct-1061259"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Zhen</forename>
                <surname>Xu</surname>
              </persName>
              <idno type="halauthorid">1635355-0</idno>
              <affiliation ref="#struct-129172"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Isabelle</forename>
                <surname>Guyon</surname>
              </persName>
              <email type="md5">159626323ec69b1da230d0f6395c7b56</email>
              <email type="domain">chalearn.org</email>
              <idno type="idhal" notation="numeric">963159</idno>
              <idno type="halauthorid" notation="string">880460-963159</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-9266-1783</idno>
              <idno type="IDREF">https://www.idref.fr/03383637X</idno>
              <affiliation ref="#struct-410475"/>
              <affiliation ref="#struct-419361"/>
              <affiliation ref="#struct-1066529"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Nihar B.</forename>
                <surname>Shah</surname>
              </persName>
              <idno type="halauthorid">1282208-0</idno>
              <affiliation ref="#struct-67135"/>
            </author>
            <editor role="depositor">
              <persName>
                <forename>Ihsan</forename>
                <surname>Ullah</surname>
              </persName>
              <email type="md5">043634345c157029250730d1706c466e</email>
              <email type="domain">gmail.com</email>
            </editor>
            <funder ref="#projanr-53256"/>
          </titleStmt>
          <editionStmt>
            <edition n="v1" type="current">
              <date type="whenSubmitted">2025-09-01 13:24:02</date>
              <date type="whenModified">2026-02-07 05:32:03</date>
              <date type="whenReleased">2025-09-01 14:20:20</date>
              <date type="whenProduced">2025-09-01</date>
              <date type="whenEndEmbargoed">2025-09-01</date>
              <ref type="file" target="https://hal.science/hal-05230379v1/document">
                <date notBefore="2025-09-01"/>
              </ref>
              <ref type="file" n="1" target="https://hal.science/hal-05230379v1/file/NeurIPS_2024_Author_Checklist.pdf" id="file-5230379-4499299">
                <date notBefore="2025-09-01"/>
              </ref>
            </edition>
            <respStmt>
              <resp>contributor</resp>
              <name key="1341603">
                <persName>
                  <forename>Ihsan</forename>
                  <surname>Ullah</surname>
                </persName>
                <email type="md5">043634345c157029250730d1706c466e</email>
                <email type="domain">gmail.com</email>
              </name>
            </respStmt>
          </editionStmt>
          <publicationStmt>
            <distributor>CCSD</distributor>
            <idno type="halId">hal-05230379</idno>
            <idno type="halUri">https://hal.science/hal-05230379</idno>
            <idno type="halBibtex">goldberg:hal-05230379</idno>
            <idno type="halRefHtml">2025</idno>
            <idno type="halRef">2025</idno>
            <availability status="restricted">
              <licence target="https://about.hal.science/hal-authorisation-v1/">HAL Authorization<ref corresp="#file-5230379-4499299"/></licence>
            </availability>
          </publicationStmt>
          <seriesStmt>
            <idno type="stamp" n="CNRS">CNRS - Centre national de la recherche scientifique</idno>
            <idno type="stamp" n="INRIA">INRIA - Institut National de Recherche en Informatique et en Automatique</idno>
            <idno type="stamp" n="INRIA-SACLAY" corresp="INRIA">INRIA Saclay - Ile de France</idno>
            <idno type="stamp" n="INRIA_TEST">INRIA - Institut National de Recherche en Informatique et en Automatique</idno>
            <idno type="stamp" n="TESTALAIN1">TESTALAIN1</idno>
            <idno type="stamp" n="CENTRALESUPELEC">Ecole CentraleSupélec</idno>
            <idno type="stamp" n="INRIA2">INRIA 2</idno>
            <idno type="stamp" n="UNIV-PARIS-SACLAY">Université Paris-Saclay</idno>
            <idno type="stamp" n="UNIVERSITE-PARIS-SACLAY" corresp="UNIV-PARIS-SACLAY">Université Paris-Saclay</idno>
            <idno type="stamp" n="ANR">ANR</idno>
            <idno type="stamp" n="LISN">Laboratoire Interdisciplinaire des Sciences du Numérique</idno>
            <idno type="stamp" n="GS-COMPUTER-SCIENCE">Graduate School Computer Science</idno>
            <idno type="stamp" n="LISN-AO" corresp="LISN">A&amp;O (Apprentissage et Optimisation)</idno>
            <idno type="stamp" n="INRIA-ETATSUNIS">Copublications Inria-Etats-Unis</idno>
            <idno type="stamp" n="LISN-STL">Sciences et Technologies des Langues</idno>
            <idno type="stamp" n="ANR-IA-19" corresp="ANR-IA">ANR-IA-19</idno>
            <idno type="stamp" n="ANR-IA">ANR-IA</idno>
            <idno type="stamp" n="LUDOSCIENCE">LUDOSCIENCE</idno>
          </seriesStmt>
          <notesStmt/>
          <sourceDesc>
            <biblStruct>
              <analytic>
                <title xml:lang="en">Usefulness of LLMs as an Author Checklist Assistant for Scientific Papers: NeurIPS’24 Experiment</title>
                <author role="aut">
                  <persName>
                    <forename type="first">Alexander</forename>
                    <surname>Goldberg</surname>
                  </persName>
                  <idno type="halauthorid">1409068-0</idno>
                  <affiliation ref="#struct-67135"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Ihsan</forename>
                    <surname>Ullah</surname>
                  </persName>
                  <idno type="halauthorid">551915-0</idno>
                  <affiliation ref="#struct-410475"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Thanh Gia Hieu</forename>
                    <surname>Khuong</surname>
                  </persName>
                  <email type="md5">4af4deff7f0f1e6756feed9d5f45d985</email>
                  <email type="domain">gmail.com</email>
                  <idno type="idhal" notation="string">ktgiahieu</idno>
                  <idno type="idhal" notation="numeric">1282340</idno>
                  <idno type="halauthorid" notation="string">2895062-1282340</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-2705-854X</idno>
                  <affiliation ref="#struct-1066529"/>
                  <affiliation ref="#struct-1061259"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Benedictus Kent</forename>
                    <surname>Rachmat</surname>
                  </persName>
                  <email type="md5">444650a22d71d5ad3aaadb51fda9ff0b</email>
                  <email type="domain">gmail.com</email>
                  <idno type="idhal" notation="string">benedictus-kent-rachmat</idno>
                  <idno type="idhal" notation="numeric">1282341</idno>
                  <idno type="halauthorid" notation="string">2895063-1282341</idno>
                  <idno type="ORCID">https://orcid.org/0009-0005-9200-7510</idno>
                  <affiliation ref="#struct-1066529"/>
                  <affiliation ref="#struct-1061259"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Zhen</forename>
                    <surname>Xu</surname>
                  </persName>
                  <idno type="halauthorid">1635355-0</idno>
                  <affiliation ref="#struct-129172"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Isabelle</forename>
                    <surname>Guyon</surname>
                  </persName>
                  <email type="md5">159626323ec69b1da230d0f6395c7b56</email>
                  <email type="domain">chalearn.org</email>
                  <idno type="idhal" notation="numeric">963159</idno>
                  <idno type="halauthorid" notation="string">880460-963159</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-9266-1783</idno>
                  <idno type="IDREF">https://www.idref.fr/03383637X</idno>
                  <affiliation ref="#struct-410475"/>
                  <affiliation ref="#struct-419361"/>
                  <affiliation ref="#struct-1066529"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Nihar B.</forename>
                    <surname>Shah</surname>
                  </persName>
                  <idno type="halauthorid">1282208-0</idno>
                  <affiliation ref="#struct-67135"/>
                </author>
              </analytic>
              <monogr>
                <imprint/>
              </monogr>
            </biblStruct>
          </sourceDesc>
          <profileDesc>
            <langUsage>
              <language ident="en">English</language>
            </langUsage>
            <textClass>
              <keywords scheme="author">
                <term xml:lang="en">peer review</term>
                <term xml:lang="en">usefulness of llms</term>
                <term xml:lang="en">checklist assistant</term>
              </keywords>
              <classCode scheme="halDomain" n="info.info-cl">Computer Science [cs]/Computation and Language [cs.CL]</classCode>
              <classCode scheme="halTypology" n="UNDEFINED">Preprints, Working Papers, ...</classCode>
              <classCode scheme="halOldTypology" n="UNDEFINED">Preprints, Working Papers, ...</classCode>
              <classCode scheme="halTreeTypology" n="UNDEFINED">Preprints, Working Papers, ...</classCode>
            </textClass>
            <abstract xml:lang="en">
              <p>Large language models (LLMs) represent a promising, but controversial, tool in aiding scientific peer review. This study evaluates the usefulness of LLMs in a conference setting as a tool for vetting paper submissions against submission standards. We conduct an experiment at the 2024 Neural Information Processing Systems (NeurIPS) conference, where 234 papers were voluntarily submitted to an "LLM-based Checklist Assistant." This assistant validates whether papers adhere to the author checklist used by NeurIPS, which includes questions to ensure compliance with research and manuscript preparation standards. Evaluation of the assistant by NeurIPS paper authors suggests that the LLM-based assistant was generally helpful in verifying checklist completion. In post-usage surveys, over 70% of authors found the assistant useful, and 70% indicate that they would revise their papers or checklist responses based on its feedback. While causal attribution to the assistant is not definitive, qualitative evidence suggests that the LLM contributed to improving some submissions. Survey responses and analysis of re-submissions indicate that authors made substantive revisions to their submissions in response to specific feedback from the LLM. The experiment also highlights common issues with LLMs: inaccuracy (20/52) and excessive strictness (14/52) were the most frequent issues flagged by authors. We also conduct experiments to understand potential gaming of the system, which reveal that the assistant could be manipulated to enhance scores through fabricated justifications, highlighting potential vulnerabilities of automated review tools.</p>
            </abstract>
          </profileDesc>
        </biblFull>
      </listBibl>
    </body>
    <back>
      <listOrg type="structures">
        <org type="institution" xml:id="struct-67135" status="VALID">
          <orgName>Carnegie Mellon University [Pittsburgh]</orgName>
          <orgName type="acronym">CMU</orgName>
          <desc>
            <address>
              <addrLine>5000 Forbes Ave,  Pittsburgh, PA 15213</addrLine>
              <country key="US"/>
            </address>
            <ref type="url">http://www.cmu.edu/</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-410475" status="VALID">
          <orgName>Chalearn</orgName>
          <desc>
            <address>
              <country key="US"/>
            </address>
            <ref type="url">http://www.chalearn.org</ref>
          </desc>
        </org>
        <org type="researchteam" xml:id="struct-1066529" status="VALID">
          <idno type="RNSR">201622258D</idno>
          <idno type="ROR">https://ror.org/031y5kf09</idno>
          <orgName>TAckling the Underspecified</orgName>
          <orgName type="acronym">TAU</orgName>
          <date type="start">2021-01-01</date>
          <date type="end">2027-12-31</date>
          <desc>
            <address>
              <addrLine>Université Paris SaclayBâtiment 660 - Shannon ; 91405 Orsay Cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.inria.fr/equipes/tau</ref>
          </desc>
          <listRelation>
            <relation active="#struct-1061259" type="direct"/>
            <relation active="#struct-300009" type="indirect"/>
            <relation active="#struct-411575" type="indirect"/>
            <relation active="#struct-419361" type="indirect"/>
            <relation name="UMR9015" active="#struct-441569" type="indirect"/>
            <relation active="#struct-1225627" type="direct"/>
            <relation active="#struct-118511" type="indirect"/>
          </listRelation>
        </org>
        <org type="laboratory" xml:id="struct-1061259" status="VALID">
          <idno type="IdRef">253160316</idno>
          <idno type="RNSR">202123711L</idno>
          <idno type="ROR">https://ror.org/00rd81916</idno>
          <idno type="Wikidata">Q109620676</idno>
          <orgName>Laboratoire Interdisciplinaire des Sciences du Numérique</orgName>
          <orgName type="acronym">LISN</orgName>
          <date type="start">2021-01-01</date>
          <desc>
            <address>
              <addrLine>Campus Universitaire bât 507Rue du Belvédère91405 Orsay cedexet Campus Universitaire bât 6401 rue Raimond Castaing91190 Gif-sur-Yvette</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.lisn.upsaclay.fr/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-300009" type="direct"/>
            <relation active="#struct-411575" type="direct"/>
            <relation active="#struct-419361" type="direct"/>
            <relation name="UMR9015" active="#struct-441569" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-129172" status="VALID">
          <idno type="ROR">https://ror.org/024mw5h28</idno>
          <orgName>University of Chicago</orgName>
          <desc>
            <address>
              <addrLine>Edward H. Levi Hall 5801 South Ellis Avenue Chicago, Illinois 60637</addrLine>
              <country key="US"/>
            </address>
            <ref type="url">http://www.uchicago.edu/</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-419361" status="VALID">
          <idno type="IdRef">241345251</idno>
          <idno type="ROR">https://ror.org/03xjwb503</idno>
          <orgName>Université Paris-Saclay</orgName>
          <desc>
            <address>
              <addrLine>Bâtiment Bréguet, 3 Rue Joliot Curie 2e ét, 91190 Gif-sur-Yvette</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.universite-paris-saclay.fr/fr</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-300009" status="VALID">
          <idno type="ROR">https://ror.org/02kvxyf05</idno>
          <orgName>Institut National de Recherche en Informatique et en Automatique</orgName>
          <orgName type="acronym">Inria</orgName>
          <desc>
            <address>
              <addrLine>Domaine de VoluceauRocquencourt - BP 10578153 Le Chesnay Cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/en/</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-411575" status="VALID">
          <idno type="IdRef">184443237</idno>
          <idno type="ROR">https://ror.org/019tcpt25</idno>
          <orgName>CentraleSupélec</orgName>
          <desc>
            <address>
              <addrLine>3, rue Joliot Curie,Plateau de Moulon,91192 GIF-SUR-YVETTE Cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.centralesupelec.fr</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-441569" status="VALID">
          <idno type="IdRef">02636817X</idno>
          <idno type="ISNI">0000000122597504</idno>
          <idno type="ROR">https://ror.org/02feahw73</idno>
          <orgName>Centre National de la Recherche Scientifique</orgName>
          <orgName type="acronym">CNRS</orgName>
          <date type="start">1939-10-19</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.cnrs.fr/</ref>
          </desc>
        </org>
        <org type="department" xml:id="struct-1225627" status="VALID">
          <idno type="ROR">https://ror.org/040753f36</idno>
          <orgName>Centre Inria de l'Université Paris-Saclay</orgName>
          <date type="start">2022-11-01</date>
          <desc>
            <address>
              <addrLine>9 Rue Joliot Curie, 91190 Gif-sur-Yvette</addrLine>
              <country key="FR"/>
            </address>
          </desc>
          <listRelation>
            <relation active="#struct-118511" type="direct"/>
            <relation active="#struct-300009" type="indirect"/>
          </listRelation>
        </org>
        <org type="laboratory" xml:id="struct-118511" status="VALID">
          <idno type="RNSR">200818248E</idno>
          <idno type="ROR">https://ror.org/0315e5x55</idno>
          <orgName>Centre Inria de Saclay</orgName>
          <desc>
            <address>
              <addrLine>1 rue Honoré d'Estienne d'OrvesBâtiment Alan TuringCampus de l'École Polytechnique91120 Palaiseau</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/centre/saclay</ref>
          </desc>
          <listRelation>
            <relation active="#struct-300009" type="direct"/>
          </listRelation>
        </org>
      </listOrg>
      <listOrg type="projects">
        <org type="anrProject" xml:id="projanr-53256" status="VALID">
          <idno type="anr">ANR-19-CHIA-0022</idno>
          <orgName>HUMANIA</orgName>
          <desc>Intelligence Artificielle pour Tous</desc>
          <date type="start">2019</date>
        </org>
      </listOrg>
    </back>
  </text>
</TEI>