index.html

<!DOCTYPE html>
<html>
  <head>
    <meta charset="utf-8" />
    <title>SWE-bench</title>
    <meta
      name="description"
      content="SWE-bench: Evaluate Language Models on Open Source Software Tasks"
    />
    <meta http-equiv="X-UA-Compatible" content="IE=edge,chrome=1" />
    <meta
      name="viewport"
      content="width=device-width, initial-scale=1, maximum-scale=1, user-scalable=no"
    />
    <meta property="og:image" content="/logo.png" />
    <link rel="shortcut icon" href="favicon.ico" type="image/x-icon" />
    <link rel="icon" href="favicon.ico" type="image/x-icon" />
    <link rel="stylesheet" href="css/normalize.css" />
    <link rel="stylesheet" href="css/fonts.css" />
    <link rel="stylesheet" href="css/styles.css" />
    <link
      rel="stylesheet"
      href="https://cdnjs.cloudflare.com/ajax/libs/font-awesome/6.2.0/css/all.min.css"
      integrity="..."
      crossorigin="anonymous"
    />
    <!-- Google tag (gtag.js) -->
    <script
      async
      src="https://www.googletagmanager.com/gtag/js?id=G-H9XFCMDPNS"
    ></script>
    <script>
      window.dataLayer = window.dataLayer || [];
      function gtag() {
        dataLayer.push(arguments);
      }
      gtag("js", new Date());

      gtag("config", "G-H9XFCMDPNS");
    </script>
  </head>
  <body>
    <div style="padding-bottom: 50px">
      <section style="background-color: var(--dark_accent_color)">
        <div
          class="content-wrapper title-wrapper"
          style="flex-direction: column"
        >
          <div
            style="
              display: flex;
              flex-direction: row;
              align-items: center;
              padding-bottom: 15px;
            "
          >
            <h1 style="font-size: 60px; padding-top: 0.4em">SWE-bench</h1>
            <img
              src="img/swellama.png"
              style="height: 100px; padding-top: 0em; padding-left: 0.5em"
            />
          </div>
          <h4>Can Language Models Resolve Real-World GitHub Issues?</h4>
          <h4 style="font-size: 20px; padding-top: 0.6em">ICLR 2024</h4>
          <p style="text-align: center;margin-top:0.5em;">
            Carlos E. Jimenez*, John Yang*, <br />
            Alexander Wettig, Shunyu Yao, Kexin Pei, <br />
            Ofir Press, Karthik Narasimhan
          </p>
          <div class="content-wrapper" style="margin-top: 2em">
            <a href="https://arxiv.org/abs/2310.06770">
              <button class="outline">
                <i class="fa fa-paperclip"></i> Paper&nbsp;
              </button>
            </a>
            <a href="https://github.com/swe-bench/SWE-bench">
              <button class="outline">
                <i class="fab fa-github"></i> Code&nbsp;
              </button>
            </a>
            <a href="submit.html">
              <button class="outline">
                <i class="fa fa-upload"></i> Submit&nbsp;
              </button>
            </a>
            <a href="viewer.html">
              <button class="outline">
                <i class="fa fa-chart-simple"></i> Analysis&nbsp;
              </button>
            </a>
          </div>
        </div>
      </section>
      <section class="main-container">
        <div class="content-wrapper" style="margin-top: 1em; display: flex; justify-content: center; align-items: center;">
          <a href="multimodal.html">
            <button
              class="outline teaser swebm"
              style="flex-direction: row; display: flex; justify-content: center; align-items: center; width: 15em; background-color: #fc959a;">
              <img
                src="img/swellamamm.png"
                style="height: 1.3em; margin-right: 0.4em; margin-bottom: 0.1em; background-color: #fc959a;" />
                SWE-bench Multimodal&nbsp;
            </button>
          </a>
          <a href="lite.html">
            <button
              class="outline teaser swebl"
              style="flex-direction: row; display: flex; justify-content: center; align-items: center; width: 15em;">
              <img
                src="img/swellama.png"
                style="height: 1.3em; margin-right: 0.4em; margin-bottom: 0.1em; background-color: var(--dark_accent_color);" />
                SWE-bench Lite&nbsp;
            </button>
          </a>
          <a href="https://openai.com/index/introducing-swe-bench-verified/">
            <button
              class="outline teaser swebv"
              style="flex-direction: row; display: flex; justify-content: center; align-items: center; width: 15em;">
              <img
                src="img/openai.png"
                style="height: 1.3em; margin-right: 0.4em; margin-bottom: 0.1em;" />
                SWE-bench Verified&nbsp;
            </button>
          </a>
        </div>
        <div class="content-wrapper">
          <div class="announcement announcement-highlight">
            <a href="https://github.com/SWE-agent/SWE-agent">
              <p class="announcement-content">
                <img src="https://avatars.githubusercontent.com/u/166046056?s=200&v=4" style="width: 1.5em; margin-right: 0.3em;"/>
                SWE-agent 1.0 is the open source SOTA on SWE-bench Lite!
              </p>
            </a>
          </div>
        </div>
        <div class="content-wrapper">
          <div class="content-box leaderboard">
            <h2 class="text-title">Leaderboard</h2>
            <ul class="tab">
              <li><button id="tab-lite" class="tablinks" data-leaderboard="Lite" style="display: flex; align-items: center;">Lite</button></li>
              <li><button id="tab-verified" class="tablinks" data-leaderboard="Verified" style="display: flex; align-items: center;">Verified</button></li>
              <li><button id="tab-test" class="tablinks" data-leaderboard="Test" style="display: flex; align-items: center;">Full</button></li>
              <li><button id="tab-multimodal" class="tablinks" data-leaderboard="Multimodal" style="display: flex; align-items: center;">Multimodal</button></li>
            </ul>
            <div class="tabcontent" style="display: block" id="leaderboard-Test">
              <table class="table scrollable">
                <thead>
                  <tr>
                    <th><div class="sticky-header-content">Model</div></th>
                    <th><div class="sticky-header-content">% Resolved</div></th>
                    <th><div class="sticky-header-content">Org</div></th>
                    <th><div class="sticky-header-content">Date</div></th>
                    <th><div class="sticky-header-content">Logs</div></th>
                    <th><div class="sticky-header-content">Trajs</div></th>
                    <th><div class="sticky-header-content">Site</div></th>
                  </tr>
                </thead>
                <tbody>
                  <tr>
                    <td>
                      <p class="model-type">
🆕 🥇 
🤠 ✅                         SWE-agent 1.0 (Claude 3.7 Sonnet)
                      </p>
                    </td>
                    <td><p class="number">33.83</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
 -                       </p>
                    </td>
                    <td><p><span class="label-date">2025-02-27</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/swe-agent/swe-agent">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🥈 
                        Amazon Q Developer Agent (v20241202-dev)
                      </p>
                    </td>
                    <td><p class="number">29.99</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://upload.wikimedia.org/wikipedia/commons/thumb/9/93/Amazon_Web_Services_Logo.svg/2560px-Amazon_Web_Services_Logo.svg.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2025-01-31</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://aws.amazon.com/q/developer/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🥉 
🤠 ✅                         OpenHands + CodeAct v2.1 (claude-3-5-sonnet-20241022)
                      </p>
                    </td>
                    <td><p class="number">29.38</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/169105795?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-11-03</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://docs.all-hands.dev/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         AutoCodeRover-v2.0 (Claude-3.5-Sonnet-20241022)
                      </p>
                    </td>
                    <td><p class="number">24.89</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://github.com/AutoCodeRoverSG/autocoderoversg.github.io/blob/main/img/logo_small.png?raw=true" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-11-21</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.autocoderover.net/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Honeycomb
                      </p>
                    </td>
                    <td><p class="number">22.06</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1826351945021636608/eQgGBRKw_400x400.jpg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-08-20</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://honeycomb.sh/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Amazon Q Developer Agent (v20240719-dev)
                      </p>
                    </td>
                    <td><p class="number">19.75</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://upload.wikimedia.org/wikipedia/commons/thumb/9/93/Amazon_Web_Services_Logo.svg/2560px-Amazon_Web_Services_Logo.svg.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-07-21</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://aws.amazon.com/q/developer/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Factory Code Droid
                      </p>
                    </td>
                    <td><p class="number">19.27</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1768168785733980160/a4hWTSfN_400x400.jpg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-06-17</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.factory.ai/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        AutoCodeRover (v20240620) + GPT 4o (2024-05-13)
                      </p>
                    </td>
                    <td><p class="number">18.83</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/100131783?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-06-28</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://autocoderover.dev/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         SWE-agent + Claude 3.5 Sonnet
                      </p>
                    </td>
                    <td><p class="number">18.13</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/166046056?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-06-20</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         AppMap Navie + GPT 4o (2024-05-13)
                      </p>
                    </td>
                    <td><p class="number">14.60</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1393240555191160833/q5OUpB3Q_400x400.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-06-15</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://appmap.io/navie">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Amazon Q Developer Agent (v20240430-dev)
                      </p>
                    </td>
                    <td><p class="number">13.82</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://upload.wikimedia.org/wikipedia/commons/thumb/9/93/Amazon_Web_Services_Logo.svg/2560px-Amazon_Web_Services_Logo.svg.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-05-09</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://aws.amazon.com/q/developer/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         SWE-agent + GPT 4 (1106)
                      </p>
                    </td>
                    <td><p class="number">12.47</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/166046056?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-04-02</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/SWE-agent/SWE-agent">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         SWE-agent + GPT 4o (2024-05-13)
                      </p>
                    </td>
                    <td><p class="number">11.99</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/166046056?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-07-28</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/SWE-agent/SWE-agent">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         SWE-agent + Claude 3 Opus
                      </p>
                    </td>
                    <td><p class="number">10.51</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/166046056?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-04-02</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         RAG + Claude 3 Opus
                      </p>
                    </td>
                    <td><p class="number">3.79</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/139597579?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-04-02</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/SWE-bench/SWE-bench/tree/main/swebench/inference">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         RAG + Claude 2
                      </p>
                    </td>
                    <td><p class="number">1.96</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/139597579?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2023-10-10</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         RAG + GPT 4 (1106)
                      </p>
                    </td>
                    <td><p class="number">1.31</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/139597579?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-04-02</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         RAG + SWE-Llama 13B
                      </p>
                    </td>
                    <td><p class="number">0.70</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/139597579?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2023-10-10</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         RAG + SWE-Llama 7B
                      </p>
                    </td>
                    <td><p class="number">0.70</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/139597579?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2023-10-10</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         RAG + ChatGPT 3.5
                      </p>
                    </td>
                    <td><p class="number">0.17</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/139597579?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2023-10-10</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                  </tr>
                </tbody>
              </table>
            </div>
            <div class="tabcontent" style="display: block" id="leaderboard-Verified">
              <table class="table scrollable">
                <thead>
                  <tr>
                    <th><div class="sticky-header-content">Model</div></th>
                    <th><div class="sticky-header-content">% Resolved</div></th>
                    <th><div class="sticky-header-content">Org</div></th>
                    <th><div class="sticky-header-content">Date</div></th>
                    <th><div class="sticky-header-content">Logs</div></th>
                    <th><div class="sticky-header-content">Trajs</div></th>
                    <th><div class="sticky-header-content">Site</div></th>
                  </tr>
                </thead>
                <tbody>
                  <tr>
                    <td>
                      <p class="model-type">
🆕 🥇 
                        Augment Agent v0
                      </p>
                    </td>
                    <td><p class="number">65.40</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://augment-assets.com/augmentcode-mark-green.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2025-03-16</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.augmentcode.com">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🥈 
                        W&B Programmer O1 crosscheck5
                      </p>
                    </td>
                    <td><p class="number">64.60</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://wandb.ai/logo.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2025-01-17</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://wandb.ai">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🥉 
                        AgentScope
                      </p>
                    </td>
                    <td><p class="number">63.40</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
 -                       </p>
                    </td>
                    <td><p><span class="label-date">2025-02-06</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/modelscope/agentscope">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🆕                         Tools + Claude 3.7 Sonnet (2025-02-24)
                      </p>
                    </td>
                    <td><p class="number">63.20</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1798110641414443008/XP8gyBaY_400x400.jpg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2025-02-24</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.anthropic.com/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🆕                         EPAM AI/Run Developer Agent v20250219 + Anthopic Claude 3.5 Sonnet
                      </p>
                    </td>
                    <td><p class="number">62.80</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://www.epam.com/content/dam/epam/homepage/epam_logo_light.svg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2025-02-28</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.epam.com/services/artificial-intelligence">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         CodeStory Midwit Agent + swe-search
                      </p>
                    </td>
                    <td><p class="number">62.20</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
 -                       </p>
                    </td>
                    <td><p><span class="label-date">2024-12-21</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://aide.dev/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         OpenHands + 4x Scaled (2024-02-03)
                      </p>
                    </td>
                    <td><p class="number">60.80</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/169105795?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2025-02-03</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://all-hands.dev/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Learn-by-interact
                      </p>
                    </td>
                    <td><p class="number">60.20</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://static-00.iconduck.com/assets.00/google-cloud-icon-2048x1646-7admxejz.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2025-01-10</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://arxiv.org/pdf/2501.10893">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        devlo
                      </p>
                    </td>
                    <td><p class="number">58.20</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://devlo.ai/static/media/devlo.ef99a045eba5ea0bcada.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-12-13</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://devlo.ai">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Emergent E1 (v2024-12-23)
                      </p>
                    </td>
                    <td><p class="number">57.20</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://framerusercontent.com/images/SgtAWOxwfupLelzIrK1vQiBWZvY.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-12-23</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://emergent.sh">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Gru(2024-12-08)
                      </p>
                    </td>
                    <td><p class="number">57.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1832709320896917504/FB5KojGT_400x400.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-12-08</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://gru.ai">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        EPAM AI/Run Developer Agent v20241212 + Anthopic Claude 3.5 Sonnet
                      </p>
                    </td>
                    <td><p class="number">55.40</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://www.epam.com/content/dam/epam/homepage/epam_logo_light.svg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-12-12</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.epam.com/services/artificial-intelligence">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Amazon Q Developer Agent (v20241202-dev)
                      </p>
                    </td>
                    <td><p class="number">55.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://upload.wikimedia.org/wikipedia/commons/thumb/9/93/Amazon_Web_Services_Logo.svg/2560px-Amazon_Web_Services_Logo.svg.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-12-02</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://aws.amazon.com/q/developer/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        devlo
                      </p>
                    </td>
                    <td><p class="number">54.20</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://devlo.ai/static/media/devlo.ef99a045eba5ea0bcada.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-11-08</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://devlo.ai">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Bracket.sh
                      </p>
                    </td>
                    <td><p class="number">53.20</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://bracket.sh/static/media/bracket_logo.d75584b91faa86a6314b.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2025-01-20</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://bracket.sh/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         OpenHands + CodeAct v2.1 (claude-3-5-sonnet-20241022)
                      </p>
                    </td>
                    <td><p class="number">53.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/169105795?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-29</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://docs.all-hands.dev/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Google Jules + Gemini 2.0 Flash (v20241212-experimental)
                      </p>
                    </td>
                    <td><p class="number">52.20</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://upload.wikimedia.org/wikipedia/commons/thumb/2/2f/Google_2015_logo.svg/2560px-Google_2015_logo.svg.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-12-12</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://developers.googleblog.com/en/the-next-chapter-of-the-gemini-era-for-developers/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Engine Labs (2024-11-25)
                      </p>
                    </td>
                    <td><p class="number">51.80</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://cdn.prod.website-files.com/669e7d9c093a7bea79b5b218/66a01c087f080328e84cb463_Engine-Icon.svg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-11-25</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.enginelabs.ai/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        AutoCodeRover-v2.1 (Claude-3.5-Sonnet-20241022)
                      </p>
                    </td>
                    <td><p class="number">51.60</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://github.com/AutoCodeRoverSG/autocoderoversg.github.io/blob/main/img/logo_small.png?raw=true" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2025-01-22</span></p></td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.autocoderover.net/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         Agentless-1.5 + Claude-3.5 Sonnet (20241022)
                      </p>
                    </td>
                    <td><p class="number">50.80</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-12-02</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/OpenAutoCoder/Agentless">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Solver (2024-10-28)
                      </p>
                    </td>
                    <td><p class="number">50.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1847427476601053184/L3RCKwMu_400x400.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-28</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://solverai.com/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Bytedance MarsCode Agent
                      </p>
                    </td>
                    <td><p class="number">50.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1768232487271153664/etPzHPm1_400x400.jpg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-11-25</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.marscode.com/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        nFactorial (2024-11-05)
                      </p>
                    </td>
                    <td><p class="number">49.20</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://nfactorial.dev/logo400.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-11-05</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://nfactorial.dev/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Tools + Claude 3.5 Sonnet (2024-10-22)
                      </p>
                    </td>
                    <td><p class="number">49.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1798110641414443008/XP8gyBaY_400x400.jpg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-22</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.anthropic.com/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         Composio SWE-Kit (2024-10-25)
                      </p>
                    </td>
                    <td><p class="number">48.60</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/128464815?s=48&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-25</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/ComposioHQ/composio/tree/master/python/swe/agent">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         AppMap Navie v2
                      </p>
                    </td>
                    <td><p class="number">47.20</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1393240555191160833/q5OUpB3Q_400x400.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-11-06</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://appmap.io/navie">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Emergent E1 (v2024-10-12)
                      </p>
                    </td>
                    <td><p class="number">46.60</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://framerusercontent.com/images/SgtAWOxwfupLelzIrK1vQiBWZvY.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-23</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://emergent.sh">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         AutoCodeRover-v2.0 (Claude-3.5-Sonnet-20241022)
                      </p>
                    </td>
                    <td><p class="number">46.20</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://github.com/AutoCodeRoverSG/autocoderoversg.github.io/blob/main/img/logo_small.png?raw=true" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-11-08</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.autocoderover.net/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Solver (2024-09-12)
                      </p>
                    </td>
                    <td><p class="number">45.40</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1847427476601053184/L3RCKwMu_400x400.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-09-24</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://laredolabs.com/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Gru(2024-08-24)
                      </p>
                    </td>
                    <td><p class="number">45.20</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1832709320896917504/FB5KojGT_400x400.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-08-24</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://gru.ai">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        CodeShellAgent + Gemini 2.0 Flash (Experimental)
                      </p>
                    </td>
                    <td><p class="number">44.20</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/145519942?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2025-01-18</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/WisdomShell/codeshell">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Solver (2024-09-12)
                      </p>
                    </td>
                    <td><p class="number">43.60</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1847427476601053184/L3RCKwMu_400x400.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-09-20</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://laredolabs.com/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🆕 🤠                         Agentless Lite + O3 Mini (20250214)
                      </p>
                    </td>
                    <td><p class="number">42.40</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2025-02-14</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/sorendunn/Agentless-Lite">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        ugaiforge
                      </p>
                    </td>
                    <td><p class="number">41.60</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
 -                       </p>
                    </td>
                    <td><p><span class="label-date">2025-01-12</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        nFactorial (2024-10-30)
                      </p>
                    </td>
                    <td><p class="number">41.60</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://nfactorial.dev/logo400.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-30</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://nfactorial.dev/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🆕 🤠                         SWE-RL (Llama3-SWE-RL-70B + Agentless Mini) (20250226)
                      </p>
                    </td>
                    <td><p class="number">41.20</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://upload.wikimedia.org/wikipedia/commons/a/ab/Meta-Logo.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2025-02-26</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/facebookresearch/swe-rl">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Nebius AI Qwen 2.5 72B Generator + LLama 3.1 70B Critic
                      </p>
                    </td>
                    <td><p class="number">40.60</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://nebius.com/favicon/favicon-96x96.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-11-13</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://nebius.com/blog/posts/training-and-search-for-software-engineering-agents">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Tools + Claude 3.5 Haiku
                      </p>
                    </td>
                    <td><p class="number">40.60</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1798110641414443008/XP8gyBaY_400x400.jpg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-22</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.anthropic.com/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Honeycomb
                      </p>
                    </td>
                    <td><p class="number">40.60</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1826351945021636608/eQgGBRKw_400x400.jpg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-08-20</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://honeycomb.sh/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         Composio SWEkit + Claude 3.5 Sonnet (2024-10-16)
                      </p>
                    </td>
                    <td><p class="number">40.60</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/128464815?s=48&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-16</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/ComposioHQ/composio/tree/master/python/swe/agent">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        EPAM AI/Run Developer Agent v20241029 + Anthopic Claude 3.5 Sonnet
                      </p>
                    </td>
                    <td><p class="number">39.60</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://www.epam.com/content/dam/epam/homepage/epam_logo_light.svg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-29</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.epam.com/services/artificial-intelligence">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Amazon Q Developer Agent (v20240719-dev)
                      </p>
                    </td>
                    <td><p class="number">38.80</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://upload.wikimedia.org/wikipedia/commons/thumb/9/93/Amazon_Web_Services_Logo.svg/2560px-Amazon_Web_Services_Logo.svg.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-07-21</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://aws.amazon.com/q/developer/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         Agentless-1.5 + GPT 4o (2024-05-13)
                      </p>
                    </td>
                    <td><p class="number">38.80</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-28</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/OpenAutoCoder/Agentless">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        AutoCodeRover (v20240620) + GPT 4o (2024-05-13)
                      </p>
                    </td>
                    <td><p class="number">38.40</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/100131783?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-06-28</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://autocoderover.dev/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         SWE-agent + Claude 3.5 Sonnet
                      </p>
                    </td>
                    <td><p class="number">33.60</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/166046056?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-06-20</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/SWE-agent/SWE-agent">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        MASAI + GPT 4o (2024-06-12)
                      </p>
                    </td>
                    <td><p class="number">32.60</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
 -                       </p>
                    </td>
                    <td><p><span class="label-date">2024-06-12</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://masai-dev-agent.github.io/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Artemis Agent v1 (2024-11-20)
                      </p>
                    </td>
                    <td><p class="number">32.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1815722024121077760/KKFl48WM_400x400.jpg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-11-20</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.turintech.ai/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        nFactorial (2024-10-07)
                      </p>
                    </td>
                    <td><p class="number">31.60</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://nfactorial.dev/logo400.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-07</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://nfactorial.dev/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         SWE-Fixer (Qwen2.5-7b retriever + Qwen2.5-72b editor) 20241128
                      </p>
                    </td>
                    <td><p class="number">30.20</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://www.shlab.org.cn/static/img/index_11.1280f447.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-11-28</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/InternLM/SWE-Fixer">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         Lingma Agent + Lingma SWE-GPT 72b (v0925)
                      </p>
                    </td>
                    <td><p class="number">28.80</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1567807256091308032/4AXfLwia_400x400.jpg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-02</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.modelscope.cn/models/yingwei/Lingma-SWE-GPT (https://www.modelscope.cn/models/yingwei/Lingma-SWE-GPT-v20240925)">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        EPAM AI/Run Developer Agent + GPT4o
                      </p>
                    </td>
                    <td><p class="number">27.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1753265803191795712/qg3YENmV_400x400.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-16</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.epam.com/services/artificial-intelligence">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         AppMap Navie + GPT 4o (2024-05-13)
                      </p>
                    </td>
                    <td><p class="number">26.20</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1393240555191160833/q5OUpB3Q_400x400.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-06-15</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://appmap.io/navie">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        nFactorial (2024-10-01)
                      </p>
                    </td>
                    <td><p class="number">25.80</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://nfactorial.dev/logo400.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-01</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://nfactorial.dev/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Amazon Q Developer Agent (v20240430-dev)
                      </p>
                    </td>
                    <td><p class="number">25.60</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://upload.wikimedia.org/wikipedia/commons/thumb/9/93/Amazon_Web_Services_Logo.svg/2560px-Amazon_Web_Services_Logo.svg.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-05-09</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://aws.amazon.com/q/developer/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         Lingma Agent + Lingma SWE-GPT 72b (v0918)
                      </p>
                    </td>
                    <td><p class="number">25.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1567807256091308032/4AXfLwia_400x400.jpg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-09-18</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.modelscope.cn/models/yingwei/Lingma-SWE-GPT">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        EPAM AI/Run Developer Agent + GPT4o
                      </p>
                    </td>
                    <td><p class="number">24.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1753265803191795712/qg3YENmV_400x400.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-08-20</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.epam.com/services/artificial-intelligence">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         SWE-agent + GPT 4o (2024-05-13)
                      </p>
                    </td>
                    <td><p class="number">23.20</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/166046056?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-07-28</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/SWE-agent/SWE-agent">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         SWE-agent + GPT 4 (1106)
                      </p>
                    </td>
                    <td><p class="number">22.40</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/166046056?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-04-02</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/SWE-agent/SWE-agent">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         SWE-agent + Claude 3 Opus
                      </p>
                    </td>
                    <td><p class="number">18.20</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/166046056?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-04-02</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/SWE-agent/SWE-agent">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         Lingma Agent + Lingma SWE-GPT 7b (v0925)
                      </p>
                    </td>
                    <td><p class="number">18.20</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1567807256091308032/4AXfLwia_400x400.jpg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-02</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.modelscope.cn/models/yingwei/Lingma-SWE-GPT (https://www.modelscope.cn/models/yingwei/Lingma-SWE-GPT-v20240925)">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         Lingma Agent + Lingma SWE-GPT 7b (v0918)
                      </p>
                    </td>
                    <td><p class="number">10.20</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1567807256091308032/4AXfLwia_400x400.jpg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-09-18</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.modelscope.cn/models/yingwei/Lingma-SWE-GPT">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         RAG + Claude 3 Opus
                      </p>
                    </td>
                    <td><p class="number">7.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/139597579?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-04-02</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/SWE-bench/SWE-bench/tree/main/swebench/inference">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         RAG + Claude 2
                      </p>
                    </td>
                    <td><p class="number">4.40</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/139597579?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2023-10-10</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         RAG + GPT 4 (1106)
                      </p>
                    </td>
                    <td><p class="number">2.80</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/139597579?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-04-02</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         RAG + SWE-Llama 7B
                      </p>
                    </td>
                    <td><p class="number">1.40</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/139597579?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2023-10-10</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         RAG + SWE-Llama 13B
                      </p>
                    </td>
                    <td><p class="number">1.20</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/139597579?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2023-10-10</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         RAG + ChatGPT 3.5
                      </p>
                    </td>
                    <td><p class="number">0.40</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/139597579?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2023-10-10</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                  </tr>
                </tbody>
              </table>
            </div>
            <div class="tabcontent" style="display: block" id="leaderboard-Lite">
              <table class="table scrollable">
                <thead>
                  <tr>
                    <th><div class="sticky-header-content">Model</div></th>
                    <th><div class="sticky-header-content">% Resolved</div></th>
                    <th><div class="sticky-header-content">Org</div></th>
                    <th><div class="sticky-header-content">Date</div></th>
                    <th><div class="sticky-header-content">Logs</div></th>
                    <th><div class="sticky-header-content">Trajs</div></th>
                    <th><div class="sticky-header-content">Site</div></th>
                  </tr>
                </thead>
                <tbody>
                  <tr>
                    <td>
                      <p class="model-type">
🥇 
                        Isoform
                      </p>
                    </td>
                    <td><p class="number">55.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/4956703?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2025-01-14</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.isoform.ai">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🥈 
                        Gru(2024-12-08)
                      </p>
                    </td>
                    <td><p class="number">48.67</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1832709320896917504/FB5KojGT_400x400.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-12-08</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://gru.ai">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🥉 
                        Globant Code Fixer Agent
                      </p>
                    </td>
                    <td><p class="number">48.33</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://ai.globant.com/wp-content/uploads/2024/06/favicon.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-11-27</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://ai.globant.com/us-en/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        devlo
                      </p>
                    </td>
                    <td><p class="number">47.33</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://devlo.ai/static/media/devlo.ef99a045eba5ea0bcada.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-11-22</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://devlo.ai">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         DARS Agent
                      </p>
                    </td>
                    <td><p class="number">47.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://drive.google.com/drive/folders/1wKpaj_37NnbSaj3OuG_FP4RDhUlKIVYM?usp=sharing" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2025-02-05</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/darsagent/DARS-Agent">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         Kodu-v1 + Claude-3.5 Sonnet (20241022)
                      </p>
                    </td>
                    <td><p class="number">44.67</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://raw.githubusercontent.com/kodu-ai/claude-coder/main/extension/assets/kodu.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-12-07</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.kodu.ai/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         OpenHands + CodeAct v2.1 (claude-3-5-sonnet-20241022)
                      </p>
                    </td>
                    <td><p class="number">41.67</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/169105795?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-25</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://docs.all-hands.dev/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         PatchKitty-0.9 + Claude-3.5 Sonnet (20241022)
                      </p>
                    </td>
                    <td><p class="number">41.33</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://brand.ucsb.edu/sites/default/files/images/pages/Visual-Identity/Marks/Seal/navy-seal.gif" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-12-20</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         OrcaLoca + Agentless-1.5 + Claude-3.5 Sonnet (20241022)
                      </p>
                    </td>
                    <td><p class="number">41.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://upload.wikimedia.org/wikipedia/commons/thumb/c/c1/Seal_of_the_University_of_California%2C_San_Diego.svg/1200px-Seal_of_the_University_of_California%2C_San_Diego.svg.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2025-01-13</span></p></td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/fishmingyu/OrcarLLM">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         Composio SWE-Kit (2024-10-30)
                      </p>
                    </td>
                    <td><p class="number">41.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/128464815?s=48&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-30</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/ComposioHQ/composio/tree/master/python/swe/agent">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         Agentless-1.5 + Claude-3.5 Sonnet (20241022)
                      </p>
                    </td>
                    <td><p class="number">40.67</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-12-02</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/OpenAutoCoder/Agentless">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        OpenCSG Starship Agentic Coder + GPT 4 (0806)
                      </p>
                    </td>
                    <td><p class="number">39.67</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://opencsg.com/images/OpenCSG_logo.svg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2025-01-13</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://opencsg.com/starship">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Bytedance MarsCode Agent
                      </p>
                    </td>
                    <td><p class="number">39.33</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1768232487271153664/etPzHPm1_400x400.jpg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-09-12</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.marscode.com/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         Moatless Tools + Claude 3.5 Sonnet (20241022)
                      </p>
                    </td>
                    <td><p class="number">39.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
 -                       </p>
                    </td>
                    <td><p><span class="label-date">2025-01-14</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/aorwall/moatless-tools">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         Moatless Tools + Claude 3.5 Sonnet (20241022)
                      </p>
                    </td>
                    <td><p class="number">38.33</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
 -                       </p>
                    </td>
                    <td><p><span class="label-date">2024-11-17</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/aorwall/moatless-tools">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Honeycomb
                      </p>
                    </td>
                    <td><p class="number">38.33</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1826351945021636608/eQgGBRKw_400x400.jpg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-08-20</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://honeycomb.sh/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         Patched.Codes Patchwork
                      </p>
                    </td>
                    <td><p class="number">37.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://imgur.com/vwVD7aJ.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2025-01-04</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/patched-codes/patchwork">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         AppMap Navie v2
                      </p>
                    </td>
                    <td><p class="number">36.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1393240555191160833/q5OUpB3Q_400x400.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-11-13</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://appmap.io/navie">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        CodeFuse-AAIS
                      </p>
                    </td>
                    <td><p class="number">35.67</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://raw.githubusercontent.com/codefuse-ai/codefuse/master/assets/LOGO.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2025-01-04</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://codefuse.ai/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Gru(2024-08-11)
                      </p>
                    </td>
                    <td><p class="number">35.67</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1832709320896917504/FB5KojGT_400x400.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-08-11</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://gru.ai">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Isoform
                      </p>
                    </td>
                    <td><p class="number">35.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
 -                       </p>
                    </td>
                    <td><p><span class="label-date">2024-08-29</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://isoform.ai">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        SuperCoder2.0
                      </p>
                    </td>
                    <td><p class="number">34.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1678659510041456640/rxUIfulT_400x400.jpg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-08-06</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://superagi.com/supercoder/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Bytedance MarsCode Agent + GPT 4o (2024-05-13)
                      </p>
                    </td>
                    <td><p class="number">34.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1768232487271153664/etPzHPm1_400x400.jpg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-07-23</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.marscode.com/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Alibaba Lingma Agent
                      </p>
                    </td>
                    <td><p class="number">33.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1567807256091308032/4AXfLwia_400x400.jpg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-06-22</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://arxiv.org/abs/2406.01422">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🆕 🤠                         Agentless Lite + O3 Mini (20250214)
                      </p>
                    </td>
                    <td><p class="number">32.33</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2025-02-14</span></p></td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/sorendunn/Agentless-Lite">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         Agentless-1.5 + GPT 4o (2024-05-13)
                      </p>
                    </td>
                    <td><p class="number">32.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-28</span></p></td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/OpenAutoCoder/Agentless">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        CodeShellTester + GPT 4o (2024-05-13)
                      </p>
                    </td>
                    <td><p class="number">31.33</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/145519942?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-11-11</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/WisdomShell/codeshell">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         AutoCodeRover (v20240620) + GPT 4o (2024-05-13)
                      </p>
                    </td>
                    <td><p class="number">30.67</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/100131783?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-06-21</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://autocoderover.dev/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         Aegis - o3-mini_1.0
                      </p>
                    </td>
                    <td><p class="number">30.33</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
 -                       </p>
                    </td>
                    <td><p><span class="label-date">2025-02-07</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/evandiewald/aegis">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        AIGCode Infant-Coder(2024-08-30)
                      </p>
                    </td>
                    <td><p class="number">30.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
 -                       </p>
                    </td>
                    <td><p><span class="label-date">2024-09-08</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://aigcode.net/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         Kortix AI (claude-3-5-sonnet-20241022)
                      </p>
                    </td>
                    <td><p class="number">30.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/170767358?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-12-03</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://www.kortix.ai/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Amazon Q Developer Agent (v20240719-dev)
                      </p>
                    </td>
                    <td><p class="number">29.67</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://upload.wikimedia.org/wikipedia/commons/thumb/9/93/Amazon_Web_Services_Logo.svg/2560px-Amazon_Web_Services_Logo.svg.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-07-21</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://aws.amazon.com/q/developer/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         Agentless + RepoGraph + GPT-4o
                      </p>
                    </td>
                    <td><p class="number">29.67</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-08-08</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/ozyyshr/RepoGraph">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        CodeR + GPT 4 (1106)
                      </p>
                    </td>
                    <td><p class="number">28.33</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://upload.wikimedia.org/wikipedia/en/thumb/0/04/Huawei_Standard_logo.svg/1200px-Huawei_Standard_logo.svg.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-06-04</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/NL2Code/CodeR">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         reproducedRG
                      </p>
                    </td>
                    <td><p class="number">28.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
 -                       </p>
                    </td>
                    <td><p><span class="label-date">2024-11-17</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        SIMA + GPT 4o (2024-05-13)
                      </p>
                    </td>
                    <td><p class="number">27.67</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
 -                       </p>
                    </td>
                    <td><p><span class="label-date">2024-07-06</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/swe-bench/experiments/tree/main/evaluation/lite/20240706_sima_gpt4o">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        MASAI + GPT 4o (2024-05-13)
                      </p>
                    </td>
                    <td><p class="number">27.33</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
 -                       </p>
                    </td>
                    <td><p><span class="label-date">2024-06-12</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://masai-dev-agent.github.io/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         Agentless + GPT 4o (2024-05-13)
                      </p>
                    </td>
                    <td><p class="number">27.33</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-06-30</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/OpenAutoCoder/Agentless">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         Moatless Tools + Claude 3.5 Sonnet
                      </p>
                    </td>
                    <td><p class="number">26.67</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
 -                       </p>
                    </td>
                    <td><p><span class="label-date">2024-06-23</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/aorwall/moatless-tools">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         OpenHands + CodeAct v1.8
                      </p>
                    </td>
                    <td><p class="number">26.67</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/169105795?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-07-25</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://docs.all-hands.dev/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        IBM Research Agent-101
                      </p>
                    </td>
                    <td><p class="number">26.67</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQPBss61ZSzmBM75JqdYCVqsfqJ4JijX0mt9g&s" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-06-12</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/swe-bench/experiments/tree/main/evaluation/lite/20240612_IBM_Research_Agent101">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         Aider + GPT 4o & Claude 3 Opus
                      </p>
                    </td>
                    <td><p class="number">26.33</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/172139148?s=48&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-05-23</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/paul-gauthier/aider">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        HyperAgent
                      </p>
                    </td>
                    <td><p class="number">25.33</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
 -                       </p>
                    </td>
                    <td><p><span class="label-date">2024-09-25</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://arxiv.org/abs/2409.16299">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         Moatless Tools + GPT 4o (2024-05-13)
                      </p>
                    </td>
                    <td><p class="number">24.67</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
 -                       </p>
                    </td>
                    <td><p><span class="label-date">2024-06-17</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/aorwall/moatless-tools">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        IBM AI Agent SWE-1.0 (with open LLMs)
                      </p>
                    </td>
                    <td><p class="number">23.67</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://encrypted-tbn0.gstatic.com/images?q=tbn:ANd9GcQPBss61ZSzmBM75JqdYCVqsfqJ4JijX0mt9g&s" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-16</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/swe-bench/experiments/tree/main/evaluation/lite/20241016_IBM-SWE-1.0">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         SWE-Fixer (Qwen2.5-7b retriever + Qwen2.5-72b editor) 20241128
                      </p>
                    </td>
                    <td><p class="number">23.33</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://www.shlab.org.cn/static/img/index_11.1280f447.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-11-28</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/InternLM/SWE-Fixer">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         SWE-agent + Claude 3.5 Sonnet
                      </p>
                    </td>
                    <td><p class="number">23.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/166046056?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-06-20</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/SWE-agent/SWE-agent">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         AppMap Navie + GPT 4o (2024-05-13)
                      </p>
                    </td>
                    <td><p class="number">21.67</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1393240555191160833/q5OUpB3Q_400x400.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-06-15</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://appmap.io/navie">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Bytedance AutoSE (based on SWE-Agent) + GPT4/GPT4o Mixed (20240828)
                      </p>
                    </td>
                    <td><p class="number">21.67</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://pbs.twimg.com/profile_images/1768232487271153664/etPzHPm1_400x400.jpg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-08-28</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
                        Amazon Q Developer Agent (v20240430-dev)
                      </p>
                    </td>
                    <td><p class="number">20.33</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://upload.wikimedia.org/wikipedia/commons/thumb/9/93/Amazon_Web_Services_Logo.svg/2560px-Amazon_Web_Services_Logo.svg.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-05-09</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://aws.amazon.com/q/developer/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠                         AutoCodeRover (v20240408) + GPT 4 (0125)
                      </p>
                    </td>
                    <td><p class="number">19.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/100131783?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-05-30</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/nus-apr/auto-code-rover">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         SWE-agent + GPT 4o (2024-05-13)
                      </p>
                    </td>
                    <td><p class="number">18.33</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/166046056?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-07-28</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/SWE-agent/SWE-agent">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         SWE-agent + GPT 4 (1106)
                      </p>
                    </td>
                    <td><p class="number">18.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/166046056?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-04-02</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/SWE-agent/SWE-agent">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         SWE-agent + Claude 3 Opus
                      </p>
                    </td>
                    <td><p class="number">11.67</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/166046056?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-04-02</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/SWE-agent/SWE-agent">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         RAG + Claude 3 Opus
                      </p>
                    </td>
                    <td><p class="number">4.33</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/139597579?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-04-02</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/SWE-bench/SWE-bench/tree/main/swebench/inference">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         RAG + Claude 2
                      </p>
                    </td>
                    <td><p class="number">3.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/139597579?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2023-10-10</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         RAG + GPT 4 (1106)
                      </p>
                    </td>
                    <td><p class="number">2.67</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/139597579?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-04-02</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         RAG + SWE-Llama 7B
                      </p>
                    </td>
                    <td><p class="number">1.33</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/139597579?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2023-10-10</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         RAG + SWE-Llama 13B
                      </p>
                    </td>
                    <td><p class="number">1.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/139597579?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2023-10-10</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         RAG + ChatGPT 3.5
                      </p>
                    </td>
                    <td><p class="number">0.33</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/139597579?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2023-10-10</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         Moatless Tools + Deepseek V3
                      </p>
                    </td>
                    <td><p class="number">0.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
 -                       </p>
                    </td>
                    <td><p><span class="label-date">2025-01-11</span></p></td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
✓                        </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/aorwall/moatless-tools">🔗</a>
                        </p>
                    </td>
                  </tr>
                </tbody>
              </table>
            </div>
            <div class="tabcontent" style="display: block" id="leaderboard-Multimodal">
              <table class="table scrollable">
                <thead>
                  <tr>
                    <th><div class="sticky-header-content">Model</div></th>
                    <th><div class="sticky-header-content">% Resolved</div></th>
                    <th><div class="sticky-header-content">Org</div></th>
                    <th><div class="sticky-header-content">Date</div></th>
                    <th><div class="sticky-header-content">Logs</div></th>
                    <th><div class="sticky-header-content">Trajs</div></th>
                    <th><div class="sticky-header-content">Site</div></th>
                  </tr>
                </thead>
                <tbody>
                  <tr>
                    <td>
                      <p class="model-type">
🆕 🥇 
✅                         Globant Code Fixer Agent
                      </p>
                    </td>
                    <td><p class="number">29.59</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://ai.globant.com/wp-content/uploads/2024/06/favicon.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2025-03-25</span></p></td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://ai.globant.com/us-en/">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🆕 🥈 
✅                         Zencoder (2025-03-10)
                      </p>
                    </td>
                    <td><p class="number">27.08</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://storage.googleapis.com/zencoder-public/zencoder-logo.svg" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2025-03-11</span></p></td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://zencoder.ai">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🆕 🥉 
🤠 ✅                         Agentless Lite + Claude-3.5 Sonnet
                      </p>
                    </td>
                    <td><p class="number">25.34</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2025-02-26</span></p></td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/sorendunn/Agentless-Lite">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         SWE-agent Multimodal + GPT 4o (2024-08-06)
                      </p>
                    </td>
                    <td><p class="number">12.19</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/166046056?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-06</span></p></td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/SWE-agent/SWE-agent">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         SWE-agent + Claude Sonnet 3.5
                      </p>
                    </td>
                    <td><p class="number">12.19</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/166046056?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-06</span></p></td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/SWE-agent/SWE-agent">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         SWE-agent JavaScript + Claude Sonnet 3.5
                      </p>
                    </td>
                    <td><p class="number">11.99</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/166046056?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-06</span></p></td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/SWE-agent/SWE-agent">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         SWE-agent + GPT 4o (2024-08-06)
                      </p>
                    </td>
                    <td><p class="number">11.99</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/166046056?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-06</span></p></td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/SWE-agent/SWE-agent">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         SWE-agent Multimodal + Claude 3.5 Sonnet
                      </p>
                    </td>
                    <td><p class="number">11.41</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/166046056?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-06</span></p></td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/SWE-agent/SWE-agent">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         SWE-agent JavaScript + GPT 4o (2024-08-06)
                      </p>
                    </td>
                    <td><p class="number">9.28</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/166046056?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-06</span></p></td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/SWE-agent/SWE-agent">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         Agentless + Claude 3.5 Sonnet
                      </p>
                    </td>
                    <td><p class="number">6.19</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-06</span></p></td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/OpenAutoCoder/Agentless">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         RAG + GPT 4o (2024-08-06)
                      </p>
                    </td>
                    <td><p class="number">6.00</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/139597579?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-06</span></p></td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/SWE-bench/SWE-bench">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         RAG + Claude 3.5 Sonnet
                      </p>
                    </td>
                    <td><p class="number">5.03</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://avatars.githubusercontent.com/u/139597579?s=200&v=4" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-06</span></p></td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/SWE-bench/SWE-bench">🔗</a>
                        </p>
                    </td>
                  </tr>
                  <tr>
                    <td>
                      <p class="model-type">
🤠 ✅                         Agentless + GPT 4o (2024-08-06)
                      </p>
                    </td>
                    <td><p class="number">3.09</p></td>
                    <td>
                      <p style="display: flex; justify-content: center; align-items: center;">
                        <img src="https://brand.illinois.edu/wp-content/uploads/2024/02/Color-Variation-Orange-Block-I-White-Background.png" style="height: 1.25em;" />
                      </p>
                    </td>
                    <td><p><span class="label-date">2024-10-06</span></p></td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
 -                         </p>
                    </td>
                    <td>
                        <p style="text-align: center;">
                            <a href="https://github.com/OpenAutoCoder/Agentless">🔗</a>
                        </p>
                    </td>
                  </tr>
                </tbody>
              </table>
            </div>
            <p class="text-content">
              SWE-bench <b>Lite</b> is a subset of SWE-bench that's been curated to make evaluation less costly and more accessible
              [<a href="lite.html">Post</a>].
              <br>
              SWE-bench <b>Verified</b> is a human annotator filtered subset that has been deemed to have a ceiling of 100% resolution rate
              [<a href="https://openai.com/index/introducing-swe-bench-verified/">Post</a>].
              <br>
              SWE-bench <b>Multimodal</b> is a new dataset featuring issues with visual elements (images, videos) from JavaScript repositories
              [<a href="multimodal.html">Post</a>]
              <br><br>
              - The <span style="color:#0ea7ff;"><b>% Resolved</b></span> metric is the percentage of instances
              (<b>2294</b> for test, <b>500</b> for verified, <b>300</b> for lite, <b>517</b> for Multimodal) <i>solved</i> by the model.
              <br>
              - <span style="color:#0ea7ff;"><b>✅ Checked</b></span> indicates that we, the SWE-bench team, received access to the system and
              were able to reproduce the patch generations.
              <br>
              - <span style="color:#0ea7ff;"><b>🤠 Open</b></span> refers to submissions that have open-source code. This does <i>not</i>
              necessarily mean the underlying model is open-source.
              <br>
              - <span style="color:#0ea7ff;"><b>🆕 New</b></span> refers to the most recently submitted solutions.
              <br><br>
              If you'd like to submit to the leaderboard, please check <a href="submit.html">this</a> page.
              All submissions are Pass@1, do not use
              <code style="color:black;background-color:#ddd;border-radius: 0.25em">hints_text</code>,
              and are in the unassisted setting.
            </p>
          </div>
        </div>
        <div class="content-wrapper" style="display: flex; justify-content: center; align-items: center;">
          <div class="content-box">
            <h2 class="text-title">News</h2>
            <p style="margin-bottom: 0.5em">
              📣 [10/2024] Introducing <b>SWE-bench Multimodal</b>! Can AI systems "see" bugs and fix them? 👀 💻
              [<a style="color:#0ca7ff" href="multimodal.html">Link</a>]
            </p>
            <p style="margin-bottom: 0.5em">
              📣 [08/2024] SWE-bench x OpenAI = <b>SWE-bench Verified</b>, a human-validated subset of 
              500 problems reviewed by software engineers!
              [<a style="color:#0ca7ff" href="https://openai.com/index/introducing-swe-bench-verified/">Report</a>]
            </p>
            <p style="margin-bottom: 0.5em">
              📣 [06/2024] We've <b>Docker</b>-ized SWE-bench for easier, containerized, reproducible evaluation.
              [<a style="color:#0ca7ff" href="https://github.com/swe-bench/SWE-bench/tree/main/docs/20240627_docker">Report</a>]
            </p>
            <p style="margin-bottom: 0.5em">
              📣 [03/2024] Check out our latest work, <b>SWE-agent</b>, which achieves a 12.47% resolve rate on SWE-bench!
              [<a href="https://github.com/princeton-nlp/SWE-agent" class="light-blue-link" target="_blank" rel="noopener noreferrer">Link</a>]
            </p>
            <p style="margin-bottom: 0.5em">
              📣 [03/2024] We've released <b>SWE-bench Lite</b>! Running all of SWE-bench can take time. This subset makes it easier!
              [<a style="color:#0ca7ff" href="lite.html">Report</a>]
            </p>
          </div>
        </div>
        <div class="content-wrapper">
          <div class="content-box">
            <h2 class="text-title">Resources</h2>
            <p class="text-content">
              You can download the SWE-bench task instances from HuggingFace or directly as a JSON
              file (<a href="https://drive.google.com/uc?export=download&id=1SbOxHiR0eXlq2azPSSOIDZz-Hva0ETpX">development</a>,
              <a href="https://drive.google.com/uc?export=download&id=164g55i3_B78F6EphCZGtgSrd2GneFyRM">test</a> sets).
              For your convenience, to fine tune your own model for evaluation on SWE-bench, we provide five pre-processed datasets at different retrieval settings ("Oracle", 13K, 27K, 40K, 50K "Llama"). We recommend using the 13K, 27K, or 40K datasets for evaluation. The 50K "Llama" dataset is provided for reproducing the results of the SWE-bench paper.
            </p>
            <div class="content-wrapper" style="width: 100%">
              <div class="content-box column">
                <a
                  style="width: 100%"
                  href="https://huggingface.co/datasets/princeton-nlp/SWE-bench"
                >
                  <div class="download">🤗 SWE-bench</div>
                </a>
                <a
                  style="width: 100%"
                  href="https://huggingface.co/datasets/princeton-nlp/SWE-bench_oracle"
                >
                  <div class="download">
                    🤗 "Oracle" Retrieval
                  </div>
                </a>
                <a
                style="width: 100%"
                href="https://huggingface.co/datasets/princeton-nlp/SWE-bench_bm25_50k_llama"
              >
                <div class="download">
                  🤗 BM25 Retrieval 50K (Llama)
                </div>
              </a>
              </div>
              <div class="content-box column">
                <a
                  style="width: 100%"
                  href="https://huggingface.co/datasets/princeton-nlp/SWE-bench_bm25_13K"
                >
                  <div class="download">
                    🤗 BM25 Retrieval 13K
                  </div>
                </a>
                <a
                  style="width: 100%"
                  href="https://huggingface.co/datasets/princeton-nlp/SWE-bench_bm25_27K"
                >
                  <div class="download">
                    🤗 BM25 Retrieval 27K
                  </div>
                </a>
                <a
                  style="width: 100%"
                  href="https://huggingface.co/datasets/princeton-nlp/SWE-bench_bm25_40K"
                >
                  <div class="download">
                    🤗 BM25 Retrieval 40K
                  </div>
                </a>
              </div>
            </div>
            <p class="text-content" style="margin-top:1em;">
              SWE-bench Lite is also available for download from HuggingFace.
            </p>
            <div class="content-wrapper" style="width: 100%">
              <div class="content-box column">
                <a
                  style="width: 100%"
                  href="https://huggingface.co/datasets/princeton-nlp/SWE-bench_Lite"
                >
                  <div class="download">🤗 SWE-bench Lite</div>
                </a>
                <a
                  style="width: 100%"
                  href="https://huggingface.co/datasets/princeton-nlp/SWE-bench_Lite_oracle"
                >
                  <div class="download">
                    🤗 "Oracle" Retrieval Lite
                  </div>
                </a>
              </div>
              <div class="content-box column">
                <a
                  style="width: 100%"
                  href="https://huggingface.co/datasets/princeton-nlp/SWE-bench_Lite_bm25_13K"
                >
                  <div class="download">
                    🤗 BM25 Retrieval 13K Lite
                  </div>
                </a>
                <a
                  style="width: 100%"
                  href="https://huggingface.co/datasets/princeton-nlp/SWE-bench_Lite_bm25_27K"
                >
                  <div class="download">
                    🤗 BM25 Retrieval 27K Lite
                  </div>
                </a>
              </div>
            </div>
            <p class="text-content" style="margin-top:1em;">
              SWE-bench Verified can be downloaded from HuggingFace.
            </p>
            <div class="content-wrapper" style="width: 100%">
              <div class="content-box column">
                <a
                  style="width: 50%"
                  href="https://huggingface.co/datasets/princeton-nlp/SWE-bench_Verified"
                >
                  <div class="download">🤗 SWE-bench Verified</div>
                </a>
              </div>
            </div>
            <p class="text-content" style="margin-top:1em;">
              We also provide the full SWE-Llama model weights at 13b and 7b parameters, along with their PEFT LoRA weights.
            </p>
            <div class="content-wrapper" style="width: 100%">
              <div class="content-box column">
                <a
                  style="width: 100%"
                  href="https://huggingface.co/princeton-nlp/SWE-Llama-13b"
                >
                  <div class="download">
                    <img
                      src="img/swellama.png"
                      style="height:1.3em;vertical-align: middle;margin-bottom:0.35em;margin-right:0.2em;background-color: var(--dark_accent_color)"
                    />
                    SWE-Llama 13b
                  </div>
                </a>
                <a
                  style="width: 100%"
                  href="https://huggingface.co/princeton-nlp/SWE-Llama-13b-peft"
                >
                  <div class="download">
                    <img
                      src="img/swellama.png"
                      style="height:1.3em;vertical-align: middle;margin-bottom:0.35em;margin-right:0.2em;background-color: var(--dark_accent_color)"
                    />
                    SWE-Llama 13b (PEFT)
                  </div>
                </a>
              </div>
              <div class="content-box column">
                <a
                  style="width: 100%"
                  href="https://huggingface.co/princeton-nlp/SWE-Llama-7b"
                >
                  <div class="download">
                    <img
                      src="img/swellama.png"
                      style="height:1.3em;vertical-align: middle;margin-bottom:0.35em;margin-right:0.2em;background-color: var(--dark_accent_color)"
                    />
                    SWE-Llama 7b
                  </div>
                </a>
                <a
                  style="width: 100%"
                  href="https://huggingface.co/princeton-nlp/SWE-Llama-7b-peft"
                >
                  <div class="download">
                    <img
                      src="img/swellama.png"
                      style="height:1.3em;vertical-align: middle;margin-bottom:0.35em;margin-right:0.2em;background-color: var(--dark_accent_color)"
                    />
                    SWE-Llama 7b (PEFT)
                  </div>
                </a>
              </div>
              
            </div>
          </div>
        </div>
        <div class="content-wrapper">
          <div class="content-box">
            <h2 class="text-title">About</h2>
            <img src="img/teaser.png" style="width:80%;margin:auto;display:block;"/>
            <p class="text-content">
              SWE-bench is a dataset that tests systems' ability to solve GitHub
              issues automatically. The dataset collects 2,294 Issue-Pull Request
              pairs from 12 popular Python repositories. Evaluation is performed by unit test verification using post-PR behavior as the reference solution.
              Read more about SWE-bench in our <a href="https://arxiv.org/abs/2310.06770", target="_blank">paper</a>!
            </p class="text-content">
            <h3 class="text-title" style="margin-bottom:0.5em">Citation</h3>
            <pre id="citation"><code>@inproceedings{
    jimenez2024swebench,
    title={{SWE}-bench: Can Language Models Resolve Real-world Github Issues?},
    author={Carlos E Jimenez and John Yang and Alexander Wettig and Shunyu Yao and Kexin Pei and Ofir Press and Karthik R Narasimhan},
    booktitle={The Twelfth International Conference on Learning Representations},
    year={2024},
    url={https://openreview.net/forum?id=VTF8yNQM66}
}</code></pre>
            <p class="text-content" style="margin-bottom: 0;">
              <b>Disclaimer:</b> SWE-bench is for research purposes only. Models
              trained and evaluated on SWE-bench can produce unexpected results.
              We are not responsible for any damages caused by the use of
              SWE-bench, including but not limited to, any loss of profit, data,
              or use of data.
            <p style="line-height: 1.6667em;">
              <b>Usage:</b> If you would like to use this website template for your
              own leaderboard, please send Carlos & John an email requesting permission.
              If granted, please make sure to acknowledge the SWE-bench team and link to
              this leaderboard on the home page of the website.
            </p>
            <p class="text-content">
              Correspondence to: <a href="mailto:carlosej@princeton.edu">carlosej@princeton.edu</a>,
              <a href="mailto:johnby@stanford.edu">johnby@stanford.edu</a>
            </p>
            <div class="content-wrapper" style="display: flex; flex-direction: row; margin-top: 0.5em;">
              <a href="https://princeton-nlp.github.io/">
                <img src="img/princeton_seal.svg" style="height: 3em;padding-top:0.5em;padding-right: 1em" />
              </a>
              <a href="https://www.cs.stanford.edu/">
                <img src="img/stanford_logo.png" style="height: 3em;padding-top:0.5em;padding-right: 1em;padding-left: 0.25em;" />
              </a>
              <a href="https://pli.princeton.edu/">
                <img src="img/pli_logo.svg" style="height: 3em;padding-top:0.5em;padding-right: 1em" />
              </a>
              <a href="https://cs.uchicago.edu/">
                <img src="img/chicago_seal.svg" style="height: 3em;padding-top:0.5em;padding-right: 1em" />
              </a>
            </div>
          </div>
        </div>
      </section>
    </div>
    <script>
      function openLeaderboard(leaderboard, updateHash = true) {
        var tabcontent = document.getElementsByClassName("tabcontent");
        for (var i = 0; i < tabcontent.length; i++) {
          tabcontent[i].style.display = "none";
        }

        var tablinks = document.getElementsByClassName("tablinks");
        for (var i = 0; i < tablinks.length; i++) {
          tablinks[i].classList.remove("active");
        }

        document.getElementById(`leaderboard-${leaderboard}`).style.display = "block";
        document.querySelector(`[data-leaderboard="${leaderboard}"]`).classList.add("active");

        // Update URL hash if requested
        if (updateHash) {
          window.location.hash = leaderboard.toLowerCase();
        }
      }

      document.addEventListener('DOMContentLoaded', function() {
        var tabs = document.querySelectorAll('.tablinks');
        tabs.forEach(function(tab) {
          tab.addEventListener('click', function(event) {
            openLeaderboard(this.getAttribute('data-leaderboard'));
            // Scroll to leaderboard section after a slight delay
            setTimeout(() => {
              document.querySelector('.leaderboard').scrollIntoView({ behavior: 'smooth' });
            }, 100);
          });
        });

        // Check URL hash for tab selection
        const hash = window.location.hash.slice(1).toLowerCase();
        if (hash === 'lite' || hash === 'verified' || hash === 'test') {
          const tabName = hash.charAt(0).toUpperCase() + hash.slice(1);
          openLeaderboard(tabName, false);
          // Scroll to leaderboard section after a slight delay
          setTimeout(() => {
            document.querySelector('.leaderboard').scrollIntoView({ behavior: 'smooth' });
          }, 100);
        } else {
          // Open the 'Lite' leaderboard by default if no hash
          openLeaderboard('Lite', false);
        }
      });
    </script>
  </body>
</html>