<?xml version="1.0" encoding="utf-8"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hal="http://hal.archives-ouvertes.fr/" xmlns:gml="http://www.opengis.net/gml/3.3/" xmlns:gmlce="http://www.opengis.net/gml/3.3/ce" version="1.1" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 http://api.archives-ouvertes.fr/documents/aofr-sword.xsd">
  <teiHeader>
    <fileDesc>
      <titleStmt>
        <title>HAL TEI export of hal-00756747</title>
      </titleStmt>
      <publicationStmt>
        <distributor>CCSD</distributor>
        <availability status="restricted">
          <licence target="https://creativecommons.org/publicdomain/zero/1.0/">CC0 1.0 - Universal</licence>
        </availability>
        <date when="2026-05-22T16:48:57+02:00"/>
      </publicationStmt>
      <sourceDesc>
        <p part="N">HAL API Platform</p>
      </sourceDesc>
    </fileDesc>
  </teiHeader>
  <text>
    <body>
      <listBibl>
        <biblFull>
          <titleStmt>
            <title xml:lang="en">A survey of actor-critic reinforcement learning: standard and natural policy gradients</title>
            <author role="crp">
              <persName>
                <forename type="first">Ivo</forename>
                <surname>Grondman</surname>
              </persName>
              <email type="md5">d755d2312943b2b8309a01253929c8e7</email>
              <email type="domain">tudelft.nl</email>
              <idno type="idhal" notation="numeric">933140</idno>
              <idno type="halauthorid" notation="string">674410-933140</idno>
              <affiliation ref="#struct-207783"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Lucian</forename>
                <surname>Busoniu</surname>
              </persName>
              <email type="md5">eec019424d15a045377a75a1ffc547d1</email>
              <email type="domain">busoniu.net</email>
              <idno type="idhal" notation="numeric">933138</idno>
              <idno type="halauthorid" notation="string">253523-933138</idno>
              <affiliation ref="#struct-185180"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Gabriel</forename>
                <surname>Lopes</surname>
              </persName>
              <email type="md5">59be2e233f2b2137b2c03969832c88bb</email>
              <email type="domain">tudelft.nl</email>
              <idno type="idhal" notation="numeric">933139</idno>
              <idno type="halauthorid" notation="string">674407-933139</idno>
              <affiliation ref="#struct-207783"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Robert</forename>
                <surname>Babuska</surname>
              </persName>
              <email type="md5">422e1a4879f9e20ebf8a8b57df7c7653</email>
              <email type="domain">tudelft.nl</email>
              <idno type="idhal" notation="numeric">933137</idno>
              <idno type="halauthorid" notation="string">253524-933137</idno>
              <affiliation ref="#struct-207783"/>
            </author>
            <editor role="depositor">
              <persName>
                <forename>Lucian</forename>
                <surname>Busoniu</surname>
              </persName>
              <email type="md5">ce13a80a143454efae13f64ff57be5c0</email>
              <email type="domain">univ-lorraine.fr</email>
            </editor>
          </titleStmt>
          <editionStmt>
            <edition n="v1" type="current">
              <date type="whenSubmitted">2012-11-23 16:24:04</date>
              <date type="whenModified">2025-11-04 12:00:42</date>
              <date type="whenReleased">2012-11-26 11:17:45</date>
              <date type="whenProduced">2012</date>
              <date type="whenEndEmbargoed">2012-11-23</date>
              <ref type="file" target="https://hal.science/hal-00756747v1/document">
                <date notBefore="2012-11-23"/>
              </ref>
              <ref type="file" subtype="author" n="1" target="https://hal.science/hal-00756747v1/file/ivo_smcc12_survey.pdf" id="file-756747-570531">
                <date notBefore="2012-11-23"/>
              </ref>
            </edition>
            <respStmt>
              <resp>contributor</resp>
              <name key="178076">
                <persName>
                  <forename>Lucian</forename>
                  <surname>Busoniu</surname>
                </persName>
                <email type="md5">ce13a80a143454efae13f64ff57be5c0</email>
                <email type="domain">univ-lorraine.fr</email>
              </name>
            </respStmt>
          </editionStmt>
          <publicationStmt>
            <distributor>CCSD</distributor>
            <idno type="halId">hal-00756747</idno>
            <idno type="halUri">https://hal.science/hal-00756747</idno>
            <idno type="halBibtex">grondman:hal-00756747</idno>
            <idno type="halRefHtml">&lt;i&gt;IEEE Transactions on Systems, Man, and Cybernetics, Part C: Applications and Reviews&lt;/i&gt;, 2012, 42 (6), pp.1291-1307. &lt;a target="_blank" href="https://dx.doi.org/10.1109/TSMCC.2012.2218595"&gt;&amp;#x27E8;10.1109/TSMCC.2012.2218595&amp;#x27E9;&lt;/a&gt;</idno>
            <idno type="halRef">IEEE Transactions on Systems, Man, and Cybernetics, Part C: Applications and Reviews, 2012, 42 (6), pp.1291-1307. &amp;#x27E8;10.1109/TSMCC.2012.2218595&amp;#x27E9;</idno>
            <availability status="restricted">
              <licence target="https://about.hal.science/hal-authorisation-v1/">HAL Authorization<ref corresp="#file-756747-570531"/></licence>
            </availability>
          </publicationStmt>
          <seriesStmt>
            <idno type="stamp" n="CNRS">CNRS - Centre national de la recherche scientifique</idno>
            <idno type="stamp" n="CRAN">Centre de Recherche en Automatique de Nancy</idno>
            <idno type="stamp" n="CRAN-ACOS" corresp="CRAN">CRAN - Automatique, commande et observations des systèmes</idno>
            <idno type="stamp" n="UNIV-LORRAINE">Université de Lorraine</idno>
            <idno type="stamp" n="TDS-MACS">Réseau de recherche en Théorie des Systèmes Distribués, Modélisation, Analyse et Contrôle des Systèmes</idno>
            <idno type="stamp" n="AM2I-UL">Pôle scientifique Automatique, Mathématiques, Informatique et leurs Intéractions de l'Université de Lorraine</idno>
          </seriesStmt>
          <notesStmt>
            <note type="audience" n="2">International</note>
            <note type="popular" n="0">No</note>
            <note type="peer" n="1">Yes</note>
          </notesStmt>
          <sourceDesc>
            <biblStruct>
              <analytic>
                <title xml:lang="en">A survey of actor-critic reinforcement learning: standard and natural policy gradients</title>
                <author role="crp">
                  <persName>
                    <forename type="first">Ivo</forename>
                    <surname>Grondman</surname>
                  </persName>
                  <email type="md5">d755d2312943b2b8309a01253929c8e7</email>
                  <email type="domain">tudelft.nl</email>
                  <idno type="idhal" notation="numeric">933140</idno>
                  <idno type="halauthorid" notation="string">674410-933140</idno>
                  <affiliation ref="#struct-207783"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Lucian</forename>
                    <surname>Busoniu</surname>
                  </persName>
                  <email type="md5">eec019424d15a045377a75a1ffc547d1</email>
                  <email type="domain">busoniu.net</email>
                  <idno type="idhal" notation="numeric">933138</idno>
                  <idno type="halauthorid" notation="string">253523-933138</idno>
                  <affiliation ref="#struct-185180"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Gabriel</forename>
                    <surname>Lopes</surname>
                  </persName>
                  <email type="md5">59be2e233f2b2137b2c03969832c88bb</email>
                  <email type="domain">tudelft.nl</email>
                  <idno type="idhal" notation="numeric">933139</idno>
                  <idno type="halauthorid" notation="string">674407-933139</idno>
                  <affiliation ref="#struct-207783"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Robert</forename>
                    <surname>Babuska</surname>
                  </persName>
                  <email type="md5">422e1a4879f9e20ebf8a8b57df7c7653</email>
                  <email type="domain">tudelft.nl</email>
                  <idno type="idhal" notation="numeric">933137</idno>
                  <idno type="halauthorid" notation="string">253524-933137</idno>
                  <affiliation ref="#struct-207783"/>
                </author>
              </analytic>
              <monogr>
                <idno type="localRef">ACOS</idno>
                <idno type="halJournalId" status="VALID">5403</idno>
                <idno type="issn">1094-6977</idno>
                <title level="j">IEEE Transactions on Systems, Man, and Cybernetics, Part C: Applications and Reviews</title>
                <imprint>
                  <publisher>Institute of Electrical and Electronics Engineers</publisher>
                  <biblScope unit="volume">42</biblScope>
                  <biblScope unit="issue">6</biblScope>
                  <biblScope unit="pp">1291-1307</biblScope>
                  <date type="datePub">2012</date>
                </imprint>
              </monogr>
              <idno type="doi">10.1109/TSMCC.2012.2218595</idno>
            </biblStruct>
          </sourceDesc>
          <profileDesc>
            <langUsage>
              <language ident="en">English</language>
            </langUsage>
            <textClass>
              <keywords scheme="author">
                <term xml:lang="en">actor-critic</term>
                <term xml:lang="en">natural gradient</term>
                <term xml:lang="en">policy gradient</term>
                <term xml:lang="en">reinforcement learning</term>
              </keywords>
              <classCode scheme="halDomain" n="spi.auto">Engineering Sciences [physics]/Automatic</classCode>
              <classCode scheme="halDomain" n="info.info-au">Computer Science [cs]/Automatic Control Engineering</classCode>
              <classCode scheme="halDomain" n="info.info-lg">Computer Science [cs]/Machine Learning [cs.LG]</classCode>
              <classCode scheme="halTypology" n="ART">Journal articles</classCode>
              <classCode scheme="halOldTypology" n="ART">Journal articles</classCode>
              <classCode scheme="halTreeTypology" n="ART">Journal articles</classCode>
            </textClass>
            <abstract xml:lang="en">
              <p>Policy gradient based actor-critic algorithms are amongst the most popular algorithms in the reinforcement learning framework. Their advantage of being able to do policy search using low-variance gradient estimates has made them useful in several real-life applications, such as robotics, power control and finance. Although general surveys on reinforcement learning techniques already exist, no survey is specifically dedicated to actor-critic algorithms in particular. This paper therefore describes the state of the art of actor-critic algorithms, with a focus on methods that can work in an online setting and use function approximation in order to deal with continuous state and action spaces. After starting with a discussion on the concepts of reinforcement learning and the origins of actor-critic algorithms, this paper describes the workings of the natural gradient, which has made its way into many actor-critic algorithms in the past few years. A review of several standard and natural actor-critic algorithms follows and the paper concludes with an overview of application areas and a discussion on open issues.</p>
            </abstract>
          </profileDesc>
        </biblFull>
      </listBibl>
    </body>
    <back>
      <listOrg type="structures">
        <org type="laboratory" xml:id="struct-207783" status="VALID">
          <orgName>Delft Center for Systems and Control [Delft]</orgName>
          <orgName type="acronym">DCSC</orgName>
          <desc>
            <address>
              <addrLine>Mekelweg 2, 2628CD Delft</addrLine>
              <country key="NL"/>
            </address>
            <ref type="url">http://www.dcsc.tudelft.nl</ref>
          </desc>
          <listRelation>
            <relation active="#struct-333368" type="direct"/>
          </listRelation>
        </org>
        <org type="laboratory" xml:id="struct-185180" status="VALID">
          <idno type="IdRef">03500472X</idno>
          <idno type="ISNI">0000000121518763</idno>
          <idno type="RNSR">200112440X</idno>
          <idno type="IdUnivLorraine">[UL]RSA--</idno>
          <idno type="ROR">https://ror.org/022r5hc56</idno>
          <orgName>Centre de Recherche en Automatique de Nancy</orgName>
          <orgName type="acronym">CRAN</orgName>
          <date type="start">2012-01-01</date>
          <desc>
            <address>
              <addrLine>Université de Lorraine, Campus Sciences, BP 70239, 54506 Vandoeuvre-les-Nancy Cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.cran.univ-lorraine.fr</ref>
          </desc>
          <listRelation>
            <relation active="#struct-413289" type="direct"/>
            <relation name="UMR7039" active="#struct-441569" type="direct"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-333368" status="VALID">
          <idno type="ROR">https://ror.org/02e2c7k09</idno>
          <orgName>Delft University of Technology</orgName>
          <orgName type="acronym">TU Delft</orgName>
          <desc>
            <address>
              <addrLine>Postbus 5,  2600 AA Delft</addrLine>
              <country key="NL"/>
            </address>
            <ref type="url">https://www.tudelft.nl/en/</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-413289" status="VALID">
          <idno type="IdRef">157040569</idno>
          <idno type="IdUnivLorraine">[UL]100--</idno>
          <idno type="ROR">https://ror.org/04vfs2w97</idno>
          <orgName>Université de Lorraine</orgName>
          <orgName type="acronym">UL</orgName>
          <date type="start">2012-01-01</date>
          <desc>
            <address>
              <addrLine>34 cours Léopold - CS 25233 - 54052 Nancy cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.univ-lorraine.fr/</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-441569" status="VALID">
          <idno type="IdRef">02636817X</idno>
          <idno type="ISNI">0000000122597504</idno>
          <idno type="ROR">https://ror.org/02feahw73</idno>
          <orgName>Centre National de la Recherche Scientifique</orgName>
          <orgName type="acronym">CNRS</orgName>
          <date type="start">1939-10-19</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.cnrs.fr/</ref>
          </desc>
        </org>
      </listOrg>
    </back>
  </text>
</TEI>