<?xml version="1.0" encoding="utf-8"?>
<TEI xmlns="http://www.tei-c.org/ns/1.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:hal="http://hal.archives-ouvertes.fr/" xmlns:gml="http://www.opengis.net/gml/3.3/" xmlns:gmlce="http://www.opengis.net/gml/3.3/ce" version="1.1" xsi:schemaLocation="http://www.tei-c.org/ns/1.0 http://api.archives-ouvertes.fr/documents/aofr-sword.xsd">
  <teiHeader>
    <fileDesc>
      <titleStmt>
        <title>HAL TEI export of hal-03509198</title>
      </titleStmt>
      <publicationStmt>
        <distributor>CCSD</distributor>
        <availability status="restricted">
          <licence target="https://creativecommons.org/publicdomain/zero/1.0/">CC0 1.0 - Universal</licence>
        </availability>
        <date when="2026-05-15T04:17:56+02:00"/>
      </publicationStmt>
      <sourceDesc>
        <p part="N">HAL API Platform</p>
      </sourceDesc>
    </fileDesc>
  </teiHeader>
  <text>
    <body>
      <listBibl>
        <biblFull>
          <titleStmt>
            <title xml:lang="en">DEISA: dask-enabled in situ analytics</title>
            <author role="aut">
              <persName>
                <forename type="first">Amal</forename>
                <surname>Gueroudji</surname>
              </persName>
              <email type="md5">23f838cc95786caad092ba2fde9e1132</email>
              <email type="domain">cea.fr</email>
              <idno type="idhal" notation="numeric">1124519</idno>
              <idno type="halauthorid" notation="string">2412959-1124519</idno>
              <orgName ref="#struct-300016"/>
              <affiliation ref="#struct-210816"/>
              <affiliation ref="#struct-1043306"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Julien</forename>
                <surname>Bigot</surname>
              </persName>
              <email type="md5">67e3a7270ae576e61283d65ef2c545df</email>
              <email type="domain">cea.fr</email>
              <idno type="idhal" notation="string">julien-bigot</idno>
              <idno type="idhal" notation="numeric">2024</idno>
              <idno type="halauthorid" notation="string">22641-2024</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-0015-4304</idno>
              <idno type="IDREF">https://www.idref.fr/154771996</idno>
              <orgName ref="#struct-300016"/>
              <affiliation ref="#struct-210816"/>
            </author>
            <author role="aut">
              <persName>
                <forename type="first">Bruno</forename>
                <surname>Raffin</surname>
              </persName>
              <email type="md5">77c302d8babe8f6345ad0fc454c8bcce</email>
              <email type="domain">inria.fr</email>
              <idno type="idhal" notation="string">bruno-raffin</idno>
              <idno type="idhal" notation="numeric">4842</idno>
              <idno type="halauthorid" notation="string">29960-4842</idno>
              <idno type="ORCID">https://orcid.org/0000-0002-7980-4946</idno>
              <idno type="IDREF">https://www.idref.fr/091616999</idno>
              <affiliation ref="#struct-1043306"/>
            </author>
            <editor role="depositor">
              <persName>
                <forename>Amal</forename>
                <surname>Gueroudji</surname>
              </persName>
              <email type="md5">77995e6f41c657fb4532ee36a93854f7</email>
              <email type="domain">gmail.com</email>
            </editor>
            <funder ref="#projeurop-713289"/>
            <funder ref="#projeurop-713842"/>
          </titleStmt>
          <editionStmt>
            <edition n="v1" type="current">
              <date type="whenSubmitted">2022-01-04 09:38:17</date>
              <date type="whenModified">2025-10-27 11:02:02</date>
              <date type="whenReleased">2022-01-04 10:37:42</date>
              <date type="whenProduced">2021-12-17</date>
              <date type="whenEndEmbargoed">2022-01-04</date>
              <ref type="file" target="https://hal.science/hal-03509198v1/document">
                <date notBefore="2022-01-04"/>
              </ref>
              <ref type="file" subtype="author" n="1" target="https://hal.science/hal-03509198v1/file/HiPC__DEISA__Dask_Enabled_In_Situ_Analytics.pdf" id="file-3509248-3073302">
                <date notBefore="2022-01-04"/>
              </ref>
            </edition>
            <respStmt>
              <resp>contributor</resp>
              <name key="1111648">
                <persName>
                  <forename>Amal</forename>
                  <surname>Gueroudji</surname>
                </persName>
                <email type="md5">77995e6f41c657fb4532ee36a93854f7</email>
                <email type="domain">gmail.com</email>
              </name>
            </respStmt>
          </editionStmt>
          <publicationStmt>
            <distributor>CCSD</distributor>
            <idno type="halId">hal-03509198</idno>
            <idno type="halUri">https://hal.science/hal-03509198</idno>
            <idno type="halBibtex">gueroudji:hal-03509198</idno>
            <idno type="halRefHtml">&lt;i&gt;HiPC 2021 - 28th International Conference on High Performance Computing, Data, and Analytics&lt;/i&gt;, Dec 2021, virtual, India. pp.1-10</idno>
            <idno type="halRef">HiPC 2021 - 28th International Conference on High Performance Computing, Data, and Analytics, Dec 2021, virtual, India. pp.1-10</idno>
            <availability status="restricted">
              <licence target="https://about.hal.science/hal-authorisation-v1/">HAL Authorization<ref corresp="#file-3509248-3073302"/></licence>
            </availability>
          </publicationStmt>
          <seriesStmt>
            <idno type="stamp" n="CEA">CEA - Commissariat à l'énergie atomique</idno>
            <idno type="stamp" n="UGA">HAL Grenoble Alpes</idno>
            <idno type="stamp" n="CNRS">CNRS - Centre national de la recherche scientifique</idno>
            <idno type="stamp" n="INRIA">INRIA - Institut National de Recherche en Informatique et en Automatique</idno>
            <idno type="stamp" n="INPG">Institut polytechnique de Grenoble</idno>
            <idno type="stamp" n="INRIA-RHA">INRIA Grenoble - Rhône-Alpes</idno>
            <idno type="stamp" n="LIG">Laboratoire d'Informatique de Grenoble</idno>
            <idno type="stamp" n="OPENAIRE">OpenAIRE</idno>
            <idno type="stamp" n="INRIA_TEST">INRIA - Institut National de Recherche en Informatique et en Automatique</idno>
            <idno type="stamp" n="LIG_SRCPR" corresp="LIG">Systèmes Répartis, Calcul Parallèle et Réseaux</idno>
            <idno type="stamp" n="TESTALAIN1">TESTALAIN1</idno>
            <idno type="stamp" n="MDLS" corresp="CEA">Maison de la Simulation</idno>
            <idno type="stamp" n="UVSQ">Université de Versailles Saint-Quentin-en-Yvelines</idno>
            <idno type="stamp" n="INRIA2">INRIA 2</idno>
            <idno type="stamp" n="CEA-UPSAY" corresp="CEA">CEA - Université Paris-Saclay</idno>
            <idno type="stamp" n="LIG-SRCPR-DATAMOVE" corresp="LIG_SRCPR">DATAMOVE - Mouvement de données pour le calcul haute performance</idno>
            <idno type="stamp" n="UNIV-PARIS-SACLAY">Université Paris-Saclay</idno>
            <idno type="stamp" n="CEA-DRF" corresp="CEA">Direction de Recherche Fondamentale</idno>
            <idno type="stamp" n="INRIA-RENGRE">INRIA-RENGRE</idno>
            <idno type="stamp" n="TEST-HALCNRS">Collection test HAL CNRS</idno>
            <idno type="stamp" n="UVSQ-UPSACLAY" corresp="UNIV-PARIS-SACLAY">UVSQ-UPSACLAY</idno>
            <idno type="stamp" n="UNIVERSITE-PARIS-SACLAY" corresp="UNIV-PARIS-SACLAY">Université Paris-Saclay</idno>
            <idno type="stamp" n="UGA-EPE">Université Grenoble Alpes [2020-*]</idno>
            <idno type="stamp" n="GS-COMPUTER-SCIENCE">Graduate School Computer Science</idno>
            <idno type="stamp" n="LIG_SIDCH" corresp="LIG">Systèmes intelligents pour les données, les connaissances et les humains</idno>
            <idno type="stamp" n="TEST-UGA">TEST-UGA</idno>
          </seriesStmt>
          <notesStmt>
            <note type="audience" n="2">International</note>
            <note type="invited" n="0">No</note>
            <note type="popular" n="0">No</note>
            <note type="peer" n="1">Yes</note>
            <note type="proceedings" n="0">No</note>
          </notesStmt>
          <sourceDesc>
            <biblStruct>
              <analytic>
                <title xml:lang="en">DEISA: dask-enabled in situ analytics</title>
                <author role="aut">
                  <persName>
                    <forename type="first">Amal</forename>
                    <surname>Gueroudji</surname>
                  </persName>
                  <email type="md5">23f838cc95786caad092ba2fde9e1132</email>
                  <email type="domain">cea.fr</email>
                  <idno type="idhal" notation="numeric">1124519</idno>
                  <idno type="halauthorid" notation="string">2412959-1124519</idno>
                  <orgName ref="#struct-300016"/>
                  <affiliation ref="#struct-210816"/>
                  <affiliation ref="#struct-1043306"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Julien</forename>
                    <surname>Bigot</surname>
                  </persName>
                  <email type="md5">67e3a7270ae576e61283d65ef2c545df</email>
                  <email type="domain">cea.fr</email>
                  <idno type="idhal" notation="string">julien-bigot</idno>
                  <idno type="idhal" notation="numeric">2024</idno>
                  <idno type="halauthorid" notation="string">22641-2024</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-0015-4304</idno>
                  <idno type="IDREF">https://www.idref.fr/154771996</idno>
                  <orgName ref="#struct-300016"/>
                  <affiliation ref="#struct-210816"/>
                </author>
                <author role="aut">
                  <persName>
                    <forename type="first">Bruno</forename>
                    <surname>Raffin</surname>
                  </persName>
                  <email type="md5">77c302d8babe8f6345ad0fc454c8bcce</email>
                  <email type="domain">inria.fr</email>
                  <idno type="idhal" notation="string">bruno-raffin</idno>
                  <idno type="idhal" notation="numeric">4842</idno>
                  <idno type="halauthorid" notation="string">29960-4842</idno>
                  <idno type="ORCID">https://orcid.org/0000-0002-7980-4946</idno>
                  <idno type="IDREF">https://www.idref.fr/091616999</idno>
                  <affiliation ref="#struct-1043306"/>
                </author>
              </analytic>
              <monogr>
                <meeting>
                  <title>HiPC 2021 - 28th International Conference on High Performance Computing, Data, and Analytics</title>
                  <date type="start">2021-12-17</date>
                  <date type="end">2021-12-17</date>
                  <settlement>virtual</settlement>
                  <country key="IN">India</country>
                </meeting>
                <imprint>
                  <publisher>IEEE</publisher>
                  <biblScope unit="pp">1-10</biblScope>
                </imprint>
              </monogr>
            </biblStruct>
          </sourceDesc>
          <profileDesc>
            <langUsage>
              <language ident="en">English</language>
            </langUsage>
            <textClass>
              <keywords scheme="author">
                <term xml:lang="en">MPI</term>
                <term xml:lang="en">Dask</term>
                <term xml:lang="en">task-based programming</term>
                <term xml:lang="en">code coupling</term>
                <term xml:lang="en">In situ processing</term>
              </keywords>
              <classCode scheme="halDomain" n="info.info-dc">Computer Science [cs]/Distributed, Parallel, and Cluster Computing [cs.DC]</classCode>
              <classCode scheme="halDomain" n="info">Computer Science [cs]</classCode>
              <classCode scheme="halDomain" n="info.info-oh">Computer Science [cs]/Other [cs.OH]</classCode>
              <classCode scheme="halTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halOldTypology" n="COMM">Conference papers</classCode>
              <classCode scheme="halTreeTypology" n="COMM">Conference papers</classCode>
            </textClass>
            <abstract xml:lang="en">
              <p>A widening performance gap is separating CPU performance and IO bandwidth on large scale systems. In some fields such as weather forecast and nuclear fusion,  numerical models generate such amounts of data that classical post hoc processing is not feasible anymore due to the limits in both storage capacity and IO performance. In situ approaches are attractive to bypass disk accesses in these cases and fully leverage the HPC platform. They are however often complex to set up and can require to re-develop parallel versions of the analysis from scratch. In this paper we propose a hybrid model that is well suited for in situ workflows that combine regular simulations and irregular analytics. Our model couples the bulk synchronous parallel paradigm for simulation with a distributed task-based one for analysis. This reduces complexity and leverages the best of each of these two powerful paradigms. We validate the model with a prototype, called DEISA, that supports coupling MPI parallel codes with analyses written using Dask. This implementation requires minimal modifications of both the simulation and analysis codes compared to their post hoc counterpart. It give access to an already existing rich ecosystem to be used in situ such as the parallel versions of Numpy, Pandas and scikit-learn. Experiments in configurations up to 1024 cores show that DEISA can improve the simulation wallclock time (excluding analysis) by a factor up to 3 and the total experiment (including analysis) hour.core cost by a factor of up to 5 compared to parallel post hoc with plain Dask while requiring the modification of only two lines of python code, three of YAML, and none at all in a C simulation code already instrumented with PDI Data Interface</p>
            </abstract>
          </profileDesc>
        </biblFull>
      </listBibl>
    </body>
    <back>
      <listOrg type="structures">
        <org type="department" xml:id="struct-210816" status="VALID">
          <idno type="IdRef">223566306</idno>
          <idno type="RNSR">201120535C</idno>
          <idno type="ROR">https://ror.org/03jv6w209</idno>
          <idno type="Wikidata">Q51780617</idno>
          <orgName>Maison de la Simulation</orgName>
          <orgName type="acronym">MDLS</orgName>
          <date type="start">2011-01-01</date>
          <desc>
            <address>
              <addrLine>USR 3441 bât. 565 CEA Saclay 91191 Gif-sur-Yvette cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.maisondelasimulation.fr/</ref>
          </desc>
          <listRelation>
            <relation name="USR3441" active="#struct-81173" type="direct"/>
            <relation active="#struct-300009" type="direct"/>
            <relation name="DRF" active="#struct-300016" type="direct"/>
            <relation active="#struct-419361" type="direct"/>
            <relation name="UAR3441 / USR3441" active="#struct-441569" type="direct"/>
          </listRelation>
        </org>
        <org type="researchteam" xml:id="struct-1043306" status="VALID">
          <idno type="RNSR">201622038P</idno>
          <idno type="ROR">https://ror.org/024cwyh09</idno>
          <orgName>Data Aware Large Scale Computing</orgName>
          <orgName type="acronym">DATAMOVE</orgName>
          <date type="start">2020-01-01</date>
          <date type="end">2026-06-30</date>
          <desc>
            <address>
              <addrLine>Batiment IMAG700 avenue CentraleDomaine Universitaire de Saint-Martin-d’HeresCS 4070038058 Grenoble cedex 9</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://team.inria.fr/datamove/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-2497" type="direct"/>
            <relation active="#struct-300009" type="indirect"/>
            <relation active="#struct-1043301" type="direct"/>
            <relation name="UMR5217" active="#struct-441569" type="indirect"/>
            <relation active="#struct-1042703" type="indirect"/>
            <relation active="#struct-1043329" type="indirect"/>
          </listRelation>
        </org>
        <org type="institution" xml:id="struct-81173" status="VALID">
          <idno type="IdRef">03082057X</idno>
          <idno type="ISNI">0000 0001 2323 0229 </idno>
          <idno type="ROR">https://ror.org/03mkjjy25</idno>
          <orgName>Université de Versailles Saint-Quentin-en-Yvelines</orgName>
          <orgName type="acronym">UVSQ</orgName>
          <date type="start">1991-07-22</date>
          <desc>
            <address>
              <addrLine>55 avenue de Paris - 78035 Versailles cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.uvsq.fr/</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-300009" status="VALID">
          <idno type="ROR">https://ror.org/02kvxyf05</idno>
          <orgName>Institut National de Recherche en Informatique et en Automatique</orgName>
          <orgName type="acronym">Inria</orgName>
          <desc>
            <address>
              <addrLine>Domaine de VoluceauRocquencourt - BP 10578153 Le Chesnay Cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/en/</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-300016" status="VALID">
          <idno type="IdRef">026372061</idno>
          <idno type="ISNI">0000000122998025</idno>
          <idno type="ROR">https://ror.org/00jjx8s55</idno>
          <idno type="Wikidata">Q868550</idno>
          <orgName>Commissariat à l'énergie atomique et aux énergies alternatives</orgName>
          <orgName type="acronym">CEA</orgName>
          <desc>
            <address>
              <addrLine>Centre de SaclayCentre de GrenobleCentre de Cadaracheetc</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.cea.fr/</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-419361" status="VALID">
          <idno type="IdRef">241345251</idno>
          <idno type="ROR">https://ror.org/03xjwb503</idno>
          <orgName>Université Paris-Saclay</orgName>
          <desc>
            <address>
              <addrLine>Bâtiment Bréguet, 3 Rue Joliot Curie 2e ét, 91190 Gif-sur-Yvette</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.universite-paris-saclay.fr/fr</ref>
          </desc>
        </org>
        <org type="regroupinstitution" xml:id="struct-441569" status="VALID">
          <idno type="IdRef">02636817X</idno>
          <idno type="ISNI">0000000122597504</idno>
          <idno type="ROR">https://ror.org/02feahw73</idno>
          <orgName>Centre National de la Recherche Scientifique</orgName>
          <orgName type="acronym">CNRS</orgName>
          <date type="start">1939-10-19</date>
          <desc>
            <address>
              <country key="FR"/>
            </address>
            <ref type="url">https://www.cnrs.fr/</ref>
          </desc>
        </org>
        <org type="laboratory" xml:id="struct-2497" status="VALID">
          <idno type="RNSR">199218244V</idno>
          <idno type="ROR">https://ror.org/00n8d6z93</idno>
          <orgName>Centre Inria de l'Université Grenoble Alpes</orgName>
          <desc>
            <address>
              <addrLine>Inovallée655 avenue de l'Europe38330 Montbonnot</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.inria.fr/centre/grenoble</ref>
          </desc>
          <listRelation>
            <relation active="#struct-300009" type="direct"/>
          </listRelation>
        </org>
        <org type="laboratory" xml:id="struct-1043301" status="VALID">
          <idno type="IdRef">148425550</idno>
          <idno type="ISNI">0000 0001 2286 4035</idno>
          <idno type="RNSR">200711886U</idno>
          <idno type="ROR">https://ror.org/01c8rcg82</idno>
          <orgName>Laboratoire d'Informatique de Grenoble</orgName>
          <orgName type="acronym">LIG</orgName>
          <date type="start">2020-01-01</date>
          <desc>
            <address>
              <addrLine>UMR 5217 - Laboratoire LIG - Bâtiment IMAG - 700 avenue Centrale - Domaine Universitaire de Saint-Martin-d’Hères Adresse postale : CS 40700 - 38058 Grenoble cedex 9Tél. : 04 57 42 14 00</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.liglab.fr/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-300009" type="direct"/>
            <relation name="UMR5217" active="#struct-441569" type="direct"/>
            <relation active="#struct-1042703" type="direct"/>
            <relation active="#struct-1043329" type="direct"/>
          </listRelation>
        </org>
        <org type="regroupinstitution" xml:id="struct-1042703" status="VALID">
          <idno type="IdRef">240648315</idno>
          <idno type="ROR">https://ror.org/02rx3b187</idno>
          <orgName>Université Grenoble Alpes</orgName>
          <orgName type="acronym">UGA</orgName>
          <date type="start">2020-01-01</date>
          <desc>
            <address>
              <addrLine>Adresse CS 40700 - 38058 Grenoble cedex</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.univ-grenoble-alpes.fr</ref>
          </desc>
        </org>
        <org type="institution" xml:id="struct-1043329" status="VALID">
          <idno type="IdRef">026388804</idno>
          <idno type="ROR">https://ror.org/05sbt2524</idno>
          <orgName>Institut polytechnique de Grenoble - Grenoble Institute of Technology</orgName>
          <orgName type="acronym">Grenoble INP</orgName>
          <date type="start">2020-01-01</date>
          <desc>
            <address>
              <addrLine>46 avenue Félix Viallet 38031 Grenoble Cedex 1</addrLine>
              <country key="FR"/>
            </address>
            <ref type="url">http://www.grenoble-inp.fr/</ref>
          </desc>
          <listRelation>
            <relation active="#struct-1042703" type="direct"/>
          </listRelation>
        </org>
      </listOrg>
      <listOrg type="projects">
        <org type="europeanProject" xml:id="projeurop-713289" status="VALID">
          <idno type="number">800945</idno>
          <idno type="program">H2020-MSCA-COFUND-2017</idno>
          <idno type="call">H2020-MSCA-COFUND-2017</idno>
          <orgName>NUMERICS</orgName>
          <desc>International PhD programme in NUMERICal Simulation</desc>
          <date type="start">2018-10-01</date>
          <date type="end">2025-03-31</date>
        </org>
        <org type="europeanProject" xml:id="projeurop-713842" status="VALID">
          <idno type="number">824158</idno>
          <idno type="program">H2020-INFRAEDI-2018-2020</idno>
          <idno type="call">H2020-INFRAEDI-2018-1</idno>
          <orgName>EoCoE-II</orgName>
          <desc>Energy Oriented Center of Excellence : toward exascale for energy</desc>
          <date type="start">2019-01-01</date>
          <date type="end">2022-06-30</date>
        </org>
      </listOrg>
    </back>
  </text>
</TEI>