2026SEL3-project-Brittle_St.../design/learning_algorithm/index.html


<!doctype html>
<html lang="en" class="no-js">
  <head>

      <meta charset="utf-8">
      <meta name="viewport" content="width=device-width,initial-scale=1">


        <link rel="prev" href="../input_action_spaces/">


        <link rel="next" href="../reward_function/">


      <link rel="icon" href="../../assets/images/favicon.png">
      <meta name="generator" content="mkdocs-1.6.1, mkdocs-material-9.7.6">


        <title>Reinforcement Learning Algorithm - Brittle Star Project</title>


      <link rel="stylesheet" href="../../assets/stylesheets/main.484c7ddc.min.css">


        <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin>
        <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,300i,400,400i,700,700i%7CRoboto+Mono:400,400i,700,700i&display=fallback">
        <style>:root{--md-text-font:"Roboto";--md-code-font:"Roboto Mono"}</style>


      <link rel="stylesheet" href="https://unpkg.com/katex@0/dist/katex.min.css">

    <script>__md_scope=new URL("../..",location),__md_hash=e=>[...e].reduce(((e,_)=>(e<<5)-e+_.charCodeAt(0)),0),__md_get=(e,_=localStorage,t=__md_scope)=>JSON.parse(_.getItem(t.pathname+"."+e)),__md_set=(e,_,t=localStorage,a=__md_scope)=>{try{t.setItem(a.pathname+"."+e,JSON.stringify(_))}catch(e){}}</script>


  </head>


    <body dir="ltr">


    <input class="md-toggle" data-md-toggle="drawer" type="checkbox" id="__drawer" autocomplete="off">
    <input class="md-toggle" data-md-toggle="search" type="checkbox" id="__search" autocomplete="off">
    <label class="md-overlay" for="__drawer"></label>
    <div data-md-component="skip">


        <a href="#reinforcement-learning-algorithm" class="md-skip">
          Skip to content
        </a>

    </div>
    <div data-md-component="announce">

    </div>


<header class="md-header md-header--shadow" data-md-component="header">
  <nav class="md-header__inner md-grid" aria-label="Header">
    <a href="../.." title="Brittle Star Project" class="md-header__button md-logo" aria-label="Brittle Star Project" data-md-component="logo">


  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>

    </a>
    <label class="md-header__button md-icon" for="__drawer">

      <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M3 6h18v2H3zm0 5h18v2H3zm0 5h18v2H3z"/></svg>
    </label>
    <div class="md-header__title" data-md-component="header-title">
      <div class="md-header__ellipsis">
        <div class="md-header__topic">
          <span class="md-ellipsis">
            Brittle Star Project
          </span>
        </div>
        <div class="md-header__topic" data-md-component="header-topic">
          <span class="md-ellipsis">

              Reinforcement Learning Algorithm

          </span>
        </div>
      </div>
    </div>


      <script>var palette=__md_get("__palette");if(palette&&palette.color){if("(prefers-color-scheme)"===palette.color.media){var media=matchMedia("(prefers-color-scheme: light)"),input=document.querySelector(media.matches?"[data-md-color-media='(prefers-color-scheme: light)']":"[data-md-color-media='(prefers-color-scheme: dark)']");palette.color.media=input.getAttribute("data-md-color-media"),palette.color.scheme=input.getAttribute("data-md-color-scheme"),palette.color.primary=input.getAttribute("data-md-color-primary"),palette.color.accent=input.getAttribute("data-md-color-accent")}for(var[key,value]of Object.entries(palette.color))document.body.setAttribute("data-md-color-"+key,value)}</script>


        <label class="md-header__button md-icon" for="__search">

          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>
        </label>
        <div class="md-search" data-md-component="search" role="dialog">
  <label class="md-search__overlay" for="__search"></label>
  <div class="md-search__inner" role="search">
    <form class="md-search__form" name="search">
      <input type="text" class="md-search__input" name="query" aria-label="Search" placeholder="Search" autocapitalize="off" autocorrect="off" autocomplete="off" spellcheck="false" data-md-component="search-query" required>
      <label class="md-search__icon md-icon" for="__search">

        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M9.5 3A6.5 6.5 0 0 1 16 9.5c0 1.61-.59 3.09-1.56 4.23l.27.27h.79l5 5-1.5 1.5-5-5v-.79l-.27-.27A6.52 6.52 0 0 1 9.5 16 6.5 6.5 0 0 1 3 9.5 6.5 6.5 0 0 1 9.5 3m0 2C7 5 5 7 5 9.5S7 14 9.5 14 14 12 14 9.5 12 5 9.5 5"/></svg>

        <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11z"/></svg>
      </label>
      <nav class="md-search__options" aria-label="Search">

        <button type="reset" class="md-search__icon md-icon" title="Clear" aria-label="Clear" tabindex="-1">

          <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M19 6.41 17.59 5 12 10.59 6.41 5 5 6.41 10.59 12 5 17.59 6.41 19 12 13.41 17.59 19 19 17.59 13.41 12z"/></svg>
        </button>
      </nav>

    </form>
    <div class="md-search__output">
      <div class="md-search__scrollwrap" tabindex="0" data-md-scrollfix>
        <div class="md-search-result" data-md-component="search-result">
          <div class="md-search-result__meta">
            Initializing search
          </div>
          <ol class="md-search-result__list" role="presentation"></ol>
        </div>
      </div>
    </div>
  </div>
</div>


  </nav>

</header>

    <div class="md-container" data-md-component="container">


      <main class="md-main" data-md-component="main">
        <div class="md-main__inner md-grid">


              <div class="md-sidebar md-sidebar--primary" data-md-component="sidebar" data-md-type="navigation" >
                <div class="md-sidebar__scrollwrap">
                  <div class="md-sidebar__inner">


<nav class="md-nav md-nav--primary" aria-label="Navigation" data-md-level="0">
  <label class="md-nav__title" for="__drawer">
    <a href="../.." title="Brittle Star Project" class="md-nav__button md-logo" aria-label="Brittle Star Project" data-md-component="logo">


  <svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M12 8a3 3 0 0 0 3-3 3 3 0 0 0-3-3 3 3 0 0 0-3 3 3 3 0 0 0 3 3m0 3.54C9.64 9.35 6.5 8 3 8v11c3.5 0 6.64 1.35 9 3.54 2.36-2.19 5.5-3.54 9-3.54V8c-3.5 0-6.64 1.35-9 3.54"/></svg>

    </a>
    Brittle Star Project
  </label>

  <ul class="md-nav__list" data-md-scrollfix>


    <li class="md-nav__item">
      <a href="../.." class="md-nav__link">


  <span class="md-ellipsis">


    Home


  </span>


      </a>
    </li>


    <li class="md-nav__item md-nav__item--active md-nav__item--nested">


        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_2" checked>


          <label class="md-nav__link" for="__nav_2" id="__nav_2_label" tabindex="0">


  <span class="md-ellipsis">


    Design & Architecture


  </span>


            <span class="md-nav__icon md-icon"></span>
          </label>

        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_2_label" aria-expanded="true">
          <label class="md-nav__title" for="__nav_2">
            <span class="md-nav__icon md-icon"></span>


    Design & Architecture


          </label>
          <ul class="md-nav__list" data-md-scrollfix>


    <li class="md-nav__item">
      <a href="../actor-critic/" class="md-nav__link">


  <span class="md-ellipsis">


    Actor-Critic Architecture


  </span>


      </a>
    </li>


    <li class="md-nav__item">
      <a href="../communication/" class="md-nav__link">


  <span class="md-ellipsis">


    Communication Scheme


  </span>


      </a>
    </li>


    <li class="md-nav__item">
      <a href="../controllers/" class="md-nav__link">


  <span class="md-ellipsis">


    Modularity & Topology


  </span>


      </a>
    </li>


    <li class="md-nav__item">
      <a href="../input_action_spaces/" class="md-nav__link">


  <span class="md-ellipsis">


    Input & Action Spaces


  </span>


      </a>
    </li>


    <li class="md-nav__item md-nav__item--active">

      <input class="md-nav__toggle md-toggle" type="checkbox" id="__toc">


        <label class="md-nav__link md-nav__link--active" for="__toc">


  <span class="md-ellipsis">


    Reinforcement Learning Algorithm


  </span>


          <span class="md-nav__icon md-icon"></span>
        </label>

      <a href="./" class="md-nav__link md-nav__link--active">


  <span class="md-ellipsis">


    Reinforcement Learning Algorithm


  </span>


      </a>


<nav class="md-nav md-nav--secondary" aria-label="Table of contents">


    <label class="md-nav__title" for="__toc">
      <span class="md-nav__icon md-icon"></span>
      Table of contents
    </label>
    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>

        <li class="md-nav__item">
  <a href="#rationale" class="md-nav__link">
    <span class="md-ellipsis">

        Rationale

    </span>
  </a>

</li>

        <li class="md-nav__item">
  <a href="#limitations-and-alternatives" class="md-nav__link">
    <span class="md-ellipsis">

        Limitations and alternatives

    </span>
  </a>

</li>

    </ul>

</nav>

    </li>


    <li class="md-nav__item">
      <a href="../reward_function/" class="md-nav__link">


  <span class="md-ellipsis">


    Reward Function & Observation Space


  </span>


      </a>
    </li>


          </ul>
        </nav>

    </li>


    <li class="md-nav__item md-nav__item--nested">


        <input class="md-nav__toggle md-toggle " type="checkbox" id="__nav_3" >


          <label class="md-nav__link" for="__nav_3" id="__nav_3_label" tabindex="0">


  <span class="md-ellipsis">


    API Reference


  </span>


            <span class="md-nav__icon md-icon"></span>
          </label>

        <nav class="md-nav" data-md-level="1" aria-labelledby="__nav_3_label" aria-expanded="false">
          <label class="md-nav__title" for="__nav_3">
            <span class="md-nav__icon md-icon"></span>


    API Reference


          </label>
          <ul class="md-nav__list" data-md-scrollfix>


    <li class="md-nav__item">
      <a href="../../api/environment/" class="md-nav__link">


  <span class="md-ellipsis">


    Brittle Star Environment


  </span>


      </a>
    </li>


    <li class="md-nav__item">
      <a href="../../api/training/" class="md-nav__link">


  <span class="md-ellipsis">


    Training Models


  </span>


      </a>
    </li>


    <li class="md-nav__item">
      <a href="../../api/tracking/" class="md-nav__link">


  <span class="md-ellipsis">


    Tracking & Monitoring


  </span>


      </a>
    </li>


    <li class="md-nav__item">
      <a href="../../api/evaluation/" class="md-nav__link">


  <span class="md-ellipsis">


    Checkpoint & Model Evaluation


  </span>


      </a>
    </li>


    <li class="md-nav__item">
      <a href="../../api/simulation/" class="md-nav__link">


  <span class="md-ellipsis">


    Interactive Simulation & Visualization


  </span>


      </a>
    </li>


    <li class="md-nav__item">
      <a href="../../api/analysis/" class="md-nav__link">


  <span class="md-ellipsis">


    Analysis & Plotting Tools


  </span>


      </a>
    </li>


    <li class="md-nav__item">
      <a href="../../api/reproduction/" class="md-nav__link">


  <span class="md-ellipsis">


    Results & Reproduction


  </span>


      </a>
    </li>


          </ul>
        </nav>

    </li>


    <li class="md-nav__item">
      <a href="../../HPC/" class="md-nav__link">


  <span class="md-ellipsis">


    HPC Guide


  </span>


      </a>
    </li>


    <li class="md-nav__item">
      <a href="../../CONTRIBUTING/" class="md-nav__link">


  <span class="md-ellipsis">


    Contribution Guidelines


  </span>


      </a>
    </li>


    <li class="md-nav__item">
      <a href="../../DEVELOPMENT/" class="md-nav__link">


  <span class="md-ellipsis">


    Development Guide


  </span>


      </a>
    </li>


  </ul>
</nav>
                  </div>
                </div>
              </div>


              <div class="md-sidebar md-sidebar--secondary" data-md-component="sidebar" data-md-type="toc" >
                <div class="md-sidebar__scrollwrap">
                  <div class="md-sidebar__inner">


<nav class="md-nav md-nav--secondary" aria-label="Table of contents">


    <label class="md-nav__title" for="__toc">
      <span class="md-nav__icon md-icon"></span>
      Table of contents
    </label>
    <ul class="md-nav__list" data-md-component="toc" data-md-scrollfix>

        <li class="md-nav__item">
  <a href="#rationale" class="md-nav__link">
    <span class="md-ellipsis">

        Rationale

    </span>
  </a>

</li>

        <li class="md-nav__item">
  <a href="#limitations-and-alternatives" class="md-nav__link">
    <span class="md-ellipsis">

        Limitations and alternatives

    </span>
  </a>

</li>

    </ul>

</nav>
                  </div>
                </div>
              </div>


            <div class="md-content" data-md-component="content">

              <article class="md-content__inner md-typeset">


<h1 id="reinforcement-learning-algorithm">Reinforcement Learning Algorithm</h1>
<p>To control the <em>continuous</em> action space (the joints of the robot) based on sensor data, we require a reliable
Reinforcement Learning (RL) algorithm or optimization strategy.</p>
<p>We have chosen <strong>Proximal Policy Optimization (PPO)</strong> (Schulman et al., 2017).</p>
<h2 id="rationale">Rationale</h2>
<p>PPO is an on-policy algorithm known for its stability and robustness (safe training without excessive variance). More
importantly, it requires relatively little hyperparameter tuning compared to other algorithms. Since NerveNet was
successfully trained using PPO (Wang et al., 2018), selecting PPO significantly reduces the risk of convergence
failures.</p>
<h2 id="limitations-and-alternatives">Limitations and alternatives</h2>
<p>Alternative learning algorithms include:</p>
<ul>
<li><strong>Twin Delayed DDPG (Fujimoto et al., 2018)</strong>: TD3 is a strong off-policy alternative used in the SMP paper (Huang et
  al., 2020). It is highly sample-efficient and reportedly excels at zero-shot adaptations. However, this approach would
  be more complex and error-prone than with PPO.</li>
<li><strong>Evolution strategies (ES)</strong>: Evolution strategies are useful for optimizing Central Pattern Generators (CPGs), e.g.
  CMA-ES, OpenAI-ES. While this method is easier to distribute and parallelize, ES typically scales worse with
  exceptionally large observation spaces compared to gradient-based RL methods like PPO.</li>
</ul>
<p><strong>References</strong></p>
<ul>
<li>Fujimoto, Scott, Herke Hoof, and David Meger. ‘Addressing Function Approximation Error in Actor-Critic Methods’. Proceedings of the 35th International Conference on Machine Learning, 3 July 2018, 1587-96. <a href="https://proceedings.mlr.press/v80/fujimoto18a.html">https://proceedings.mlr.press/v80/fujimoto18a.html</a>.</li>
<li>Huang, Wenlong, Igor Mordatch, and Deepak Pathak. ‘One Policy to Control Them All: Shared Modular Policies for Agent-Agnostic Control’. arXiv:2007.04976. Preprint, arXiv, 9 July 2020. <a href="https://doi.org/10.48550/arXiv.2007.04976">https://doi.org/10.48550/arXiv.2007.04976</a>.</li>
<li>Schulman, John, Filip Wolski, Prafulla Dhariwal, Alec Radford, and Oleg Klimov. ‘Proximal Policy Optimization Algorithms’. arXiv:1707.06347. Preprint, arXiv, 28 August 2017. <a href="https://doi.org/10.48550/arXiv.1707.06347">https://doi.org/10.48550/arXiv.1707.06347</a>.</li>
</ul>


              </article>
            </div>


<script>var target=document.getElementById(location.hash.slice(1));target&&target.name&&(target.checked=target.name.startsWith("__tabbed_"))</script>
        </div>

      </main>

        <footer class="md-footer">

  <div class="md-footer-meta md-typeset">
    <div class="md-footer-meta__inner md-grid">
      <div class="md-copyright">


    Made with
    <a href="https://squidfunk.github.io/mkdocs-material/" target="_blank" rel="noopener">
      Material for MkDocs
    </a>

</div>

    </div>
  </div>
</footer>

    </div>
    <div class="md-dialog" data-md-component="dialog">
      <div class="md-dialog__inner md-typeset"></div>
    </div>


      <script id="__config" type="application/json">{"annotate": null, "base": "../..", "features": [], "search": "../../assets/javascripts/workers/search.2c215733.min.js", "tags": null, "translations": {"clipboard.copied": "Copied to clipboard", "clipboard.copy": "Copy to clipboard", "search.result.more.one": "1 more on this page", "search.result.more.other": "# more on this page", "search.result.none": "No matching documents", "search.result.one": "1 matching document", "search.result.other": "# matching documents", "search.result.placeholder": "Type to start searching", "search.result.term.missing": "Missing", "select.version": "Select version"}, "version": null}</script>


      <script src="../../assets/javascripts/bundle.79ae519e.min.js"></script>

        <script src="../../javascripts/katex.js"></script>

        <script src="https://unpkg.com/katex@0/dist/katex.min.js"></script>

        <script src="https://unpkg.com/katex@0/dist/contrib/auto-render.min.js"></script>


  </body>
</html>