Merge branch 'main' into openhands-fix-issue-5076

.github/workflows/lint-fix.yml

@@ -16,6 +16,7 @@ jobs:
       - uses: actions/checkout@v4
         with:
           ref: ${{ github.head_ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name }}
           fetch-depth: 0
           token: ${{ secrets.GITHUB_TOKEN }}
 
@@ -43,7 +44,8 @@ jobs:
         run: pip install pre-commit==3.7.0
       - name: Fix python lint issues
         run: |
-          pre-commit run --files openhands/**/* evaluation/**/* tests/**/* --config ./dev_config/python/.pre-commit-config.yaml --all-files
+          # Run all pre-commit hooks and continue even if they modify files (exit code 1)
+          pre-commit run --config ./dev_config/python/.pre-commit-config.yaml --files openhands/**/* evaluation/**/* tests/**/* || true
 
       # Commit and push changes if any
       - name: Check for changes

.github/workflows/openhands-resolver.yml

@@ -11,6 +11,11 @@ on:
         required: false
         type: string
         default: "@openhands-agent"
+      target_branch:
+        required: false
+        type: string
+        default: "main"
+        description: "Target branch to pull and create PR against"
     secrets:
       LLM_MODEL:
         required: true
@@ -40,7 +45,6 @@ permissions:
   issues: write
 
 jobs:
-
   auto-fix:
     if: |
       github.event_name == 'workflow_call' ||
@@ -76,7 +80,18 @@ jobs:
           cat requirements.txt
 
       - name: Cache pip dependencies
-        if: github.event.label.name != 'fix-me-experimental'
+        if: |
+          !(
+            github.event.label.name == 'fix-me-experimental' ||
+            (
+              (github.event_name == 'issue_comment' || github.event_name == 'pull_request_review_comment') &&
+              startsWith(github.event.comment.body, '@openhands-agent-exp')
+            ) ||
+            (
+              github.event_name == 'pull_request_review' &&
+              startsWith(github.event.review.body, '@openhands-agent-exp')
+            )
+          )
         uses: actions/cache@v3
         with:
           path: ${{ env.pythonLocation }}/lib/python3.12/site-packages/*
@@ -125,6 +140,9 @@ jobs:
           echo "MAX_ITERATIONS=${{ inputs.max_iterations || 50 }}" >> $GITHUB_ENV
           echo "SANDBOX_ENV_GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> $GITHUB_ENV
 
+          # Set branch variables
+          echo "TARGET_BRANCH=${{ inputs.target_branch }}" >> $GITHUB_ENV
+
       - name: Comment on issue with start message
         uses: actions/github-script@v7
         with:
@@ -140,7 +158,11 @@ jobs:
 
       - name: Install OpenHands
         run: |
-          if [ "${{ github.event.label.name }}" == "fix-me-experimental" ]; then
+          if [[ "${{ github.event.label.name }}" == "fix-me-experimental" ]] ||
+             ([[ "${{ github.event_name }}" == "issue_comment" || "${{ github.event_name }}" == "pull_request_review_comment" ]] &&
+              [[ "${{ github.event.comment.body }}" == "@openhands-agent-exp"* ]]) ||
+             ([[ "${{ github.event_name }}" == "pull_request_review" ]] &&
+              [[ "${{ github.event.review.body }}" == "@openhands-agent-exp"* ]]); then
             python -m pip install --upgrade pip
             pip install git+https://github.com/all-hands-ai/openhands.git
           else
@@ -161,7 +183,7 @@ jobs:
             --issue-number ${{ env.ISSUE_NUMBER }} \
             --issue-type ${{ env.ISSUE_TYPE }} \
             --max-iterations ${{ env.MAX_ITERATIONS }} \
-            --comment-id ${{ env.COMMENT_ID }}
+            --comment-id ${{ env.COMMENT_ID }} \
 
       - name: Check resolution result
         id: check_result

.github/workflows/run-eval.yml

@@ -0,0 +1,53 @@
+# Run evaluation on a PR
+name: Run Eval
+
+# Runs when a PR is labeled with one of the "run-eval-" labels
+on:
+  pull_request:
+    types: [labeled]
+
+jobs:
+  trigger-job:
+    name: Trigger remote eval job
+    if: ${{ github.event.label.name == 'run-eval-xs' || github.event.label.name == 'run-eval-s' || github.event.label.name == 'run-eval-m' }}
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout PR branch
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ github.head_ref }}
+
+      - name: Trigger remote job
+        run: |
+          REPO_URL="https://github.com/${{ github.repository }}"
+          PR_BRANCH="${{ github.head_ref }}"
+          echo "Repository URL: $REPO_URL"
+          echo "PR Branch: $PR_BRANCH"
+
+          if [[ "${{ github.event.label.name }}" == "run-eval-xs" ]]; then
+            EVAL_INSTANCES="1"
+          elif [[ "${{ github.event.label.name }}" == "run-eval-s" ]]; then
+            EVAL_INSTANCES="5"
+          elif [[ "${{ github.event.label.name }}" == "run-eval-m" ]]; then
+            EVAL_INSTANCES="30"
+          fi
+
+          curl -X POST \
+            -H "Authorization: Bearer ${{ secrets.PAT_TOKEN }}" \
+            -H "Accept: application/vnd.github+json" \
+            -d "{\"ref\": \"main\", \"inputs\": {\"github-repo\": \"${REPO_URL}\", \"github-branch\": \"${PR_BRANCH}\", \"pr-number\": \"${{ github.event.pull_request.number }}\", \"eval-instances\": \"${EVAL_INSTANCES}\"}}" \
+            https://api.github.com/repos/All-Hands-AI/evaluation/actions/workflows/create-branch.yml/dispatches
+
+          # Send Slack message
+          PR_URL="https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}"
+          slack_text="PR $PR_URL has triggered evaluation on $EVAL_INSTANCES instances..."
+          curl -X POST -H 'Content-type: application/json' --data '{"text":"'"$slack_text"'"}' \
+            https://hooks.slack.com/services/${{ secrets.SLACK_TOKEN }}
+
+      - name: Comment on PR
+        uses: KeisukeYamashita/create-comment@v1
+        with:
+          unique: false
+          comment: |
+            Running evaluation on the PR. Once eval is done, the results will be posted.

.gitignore

@@ -175,6 +175,7 @@ evaluation/gaia/data
 evaluation/gorilla/data
 evaluation/toolqa/data
 evaluation/scienceagentbench/benchmark
+evaluation/commit0_bench/repos
 
 # openhands resolver
 output/

COMMUNITY.md

@@ -0,0 +1,43 @@
+# 🙌 The OpenHands Community
+
+The OpenHands community is built around the belief that (1) AI and AI agents are going to fundamentally change the way
+we build software, and (2) if this is true, we should do everything we can to make sure that the benefits provided by
+such powerful technology are accessible to everyone.
+
+If this resonates with you, we'd love to have you join us in our quest!
+
+## 🤝 How to Join
+
+Check out our [How to Join the Community section.](https://github.com/All-Hands-AI/OpenHands?tab=readme-ov-file#-how-to-join-the-community)
+
+## 💪 Becoming a Contributor
+
+We welcome contributions from everyone! Whether you're a developer, a researcher, or simply enthusiastic about advancing
+the field of software engineering with AI, there are many ways to get involved:
+
+- **Code Contributions:** Help us develop new core functionality, improve our agents, improve the frontend and other
+interfaces, or anything else that would help make OpenHands better.
+- **Research and Evaluation:** Contribute to our understanding of LLMs in software engineering, participate in
+evaluating the models, or suggest improvements.
+- **Feedback and Testing:** Use the OpenHands toolset, report bugs, suggest features, or provide feedback on usability.
+
+For details, please check [CONTRIBUTING.md](./CONTRIBUTING.md).
+
+## Code of Conduct
+
+We have a [Code of Conduct](./CODE_OF_CONDUCT.md) that we expect all contributors to adhere to.
+Long story short, we are aiming for an open, welcoming, diverse, inclusive, and healthy community.
+All contributors are expected to contribute to building this sort of community.
+
+## 🛠️ Becoming a Maintainer
+
+For contributors who have made significant and sustained contributions to the project, there is a possibility of joining
+the maintainer team. The process for this is as follows:
+
+1. Any contributor who has made sustained and high-quality contributions to the codebase can be nominated by any
+maintainer. If you feel that you may qualify you can reach out to any of the maintainers that have reviewed your PRs and ask if you can be nominated.
+2. Once a maintainer nominates a new maintainer, there will be a discussion period among the maintainers for at least 3 days.
+3. If no concerns are raised the nomination will be accepted by acclamation, and if concerns are raised there will be a discussion and possible vote.
+
+Note that just making many PRs does not immediately imply that you will become a maintainer. We will be looking
+at sustained high-quality contributions over a period of time, as well as good teamwork and adherence to our [Code of Conduct](./CODE_OF_CONDUCT.md).

CONTRIBUTING.md

@@ -54,7 +54,7 @@ The agent needs a place to run code and commands. When you run OpenHands on your
 to do this by default. But there are other ways of creating a sandbox for the agent.
 
 If you work for a company that provides a cloud-based runtime, you could help us add support for that runtime
-by implementing the [interface specified here](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/runtime/runtime.py).
+by implementing the [interface specified here](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/runtime/base.py).
 
 #### Testing
 When you write code, it is also good to write tests. Please navigate to the `tests` folder to see existing test suites.

README.md

@@ -77,28 +77,21 @@ To learn more about the project, and for tips on using OpenHands,
 There you'll find resources on how to use different LLM providers,
 troubleshooting resources, and advanced configuration options.
 
-## 🤝 How to Contribute
+## 🤝 How to Join the Community
 
-OpenHands is a community-driven project, and we welcome contributions from everyone.
-Whether you're a developer, a researcher, or simply enthusiastic about advancing the field of
-software engineering with AI, there are many ways to get involved:
+OpenHands is a community-driven project, and we welcome contributions from everyone. We do most of our communication
+through Slack, so this is the best place to start, but we also are happy to have you contact us on Discord or Github:
 
-- **Code Contributions:** Help us develop new agents, core functionality, the frontend and other interfaces, or sandboxing solutions.
-- **Research and Evaluation:** Contribute to our understanding of LLMs in software engineering, participate in evaluating the models, or suggest improvements.
-- **Feedback and Testing:** Use the OpenHands toolset, report bugs, suggest features, or provide feedback on usability.
+- [Join our Slack workspace](https://join.slack.com/t/openhands-ai/shared_invite/zt-2tom0er4l-JeNUGHt_AxpEfIBstbLPiw) - Here we talk about research, architecture, and future development.
+- [Join our Discord server](https://discord.gg/ESHStjSjD4) - This is a community-run server for general discussion, questions, and feedback.
+- [Read or post Github Issues](https://github.com/All-Hands-AI/OpenHands/issues) - Check out the issues we're working on, or add your own ideas.
 
-For details, please check [CONTRIBUTING.md](./CONTRIBUTING.md).
-
-## 🤖 Join Our Community
-
-Whether you're a developer, a researcher, or simply enthusiastic about OpenHands, we'd love to have you in our community.
-Let's make software engineering better together!
-
-- [Slack workspace](https://join.slack.com/t/openhands-ai/shared_invite/zt-2tom0er4l-JeNUGHt_AxpEfIBstbLPiw) - Here we talk about research, architecture, and future development.
-- [Discord server](https://discord.gg/ESHStjSjD4) - This is a community-run server for general discussion, questions, and feedback.
+See more about the community in [COMMUNITY.md](./COMMUNITY.md) or find details on contributing in [CONTRIBUTING.md](./CONTRIBUTING.md).
 
 ## 📈 Progress
 
+See the monthly OpenHands roadmap [here](https://github.com/orgs/All-Hands-AI/projects/1) (updated at the maintainer's meeting at the end of each month).
+
 <p align="center">
   <a href="https://star-history.com/#All-Hands-AI/OpenHands&Date">
     <img src="https://api.star-history.com/svg?repos=All-Hands-AI/OpenHands&type=Date" width="500" alt="Star History Chart">

docs/i18n/fr/docusaurus-plugin-content-docs/current/usage/how-to/github-action.md

@@ -14,4 +14,4 @@ Pour utiliser l'Action GitHub OpenHands dans le dépôt OpenHands, un mainteneur
 
 ## Installation de l'Action dans un nouveau dépôt
 
-Pour installer l'Action GitHub OpenHands dans votre propre dépôt, suivez les [instructions dans le dépôt OpenHands Resolver](https://github.com/All-Hands-AI/OpenHands-resolver?tab=readme-ov-file#using-the-github-actions-workflow).
+Pour installer l'Action GitHub OpenHands dans votre propre dépôt, suivez les [instructions dans le dépôt OpenHands Resolver](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/resolver/README.md).

docs/i18n/zh-Hans/docusaurus-plugin-content-docs/current/usage/how-to/github-action.md

@@ -12,4 +12,4 @@
 
 ## 在新仓库中安装 Action
 
-要在你自己的仓库中安装 OpenHands GitHub Action，请按照 [OpenHands Resolver 仓库中的说明](https://github.com/All-Hands-AI/OpenHands-resolver?tab=readme-ov-file#using-the-github-actions-workflow) 进行操作。
+要在你自己的仓库中安装 OpenHands GitHub Action，请按照 [OpenHands Resolver 仓库中的说明](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/resolver/README.md) 进行操作。

docs/modules/usage/about.md

@@ -21,9 +21,6 @@ OpenHands is built using a combination of powerful frameworks and libraries, pro
 
 Please note that the selection of these technologies is in progress, and additional technologies may be added or existing ones may be removed as the project evolves. We strive to adopt the most suitable and efficient tools to enhance the capabilities of OpenHands.
 
-## Licensing, Contributing, Community Servers
+## License
 
 Distributed under MIT [License](https://github.com/All-Hands-AI/OpenHands/blob/main/LICENSE).
-
-For guides on how to contribute to OpenHands, joining our Discord and Slack servers
-[check out the OpenHands README.](https://github.com/All-Hands-AI/OpenHands?tab=readme-ov-file#-how-to-contribute)

docs/modules/usage/how-to/github-action.md

@@ -4,13 +4,92 @@ This guide explains how to use the OpenHands GitHub Action, both within the Open
 
 ## Using the Action in the OpenHands Repository
 
-To use the OpenHands GitHub Action in the OpenHands repository, an OpenHands maintainer can:
+To use the OpenHands GitHub Action in a repository, you can:
 
 1. Create an issue in the repository.
-2. Add the `fix-me` label to the issue.
-3. The action will automatically trigger and attempt to resolve the issue.
+2. Add the `fix-me` label to the issue or leave a comment on the issue starting with `@openhands-agent`.
+
+The action will automatically trigger and attempt to resolve the issue.
 
 ## Installing the Action in a New Repository
 
 To install the OpenHands GitHub Action in your own repository, follow
 the [README for the OpenHands Resolver](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/resolver/README.md).
+
+## Usage Tips
+
+### Iterative resolution
+
+1. Create an issue in the repository.
+2. Add the `fix-me` label to the issue, or leave a comment starting with `@openhands-agent`
+3. Review the attempt to resolve the issue by checking the pull request
+4. Follow up with feedback through general comments, review comments, or inline thread comments
+5. Add the `fix-me` label to the pull request, or address a specific comment by starting with `@openhands-agent`
+
+### Label versus Macro
+
+- Label (`fix-me`): Requests OpenHands to address the **entire** issue or pull request.
+- Macro (`@openhands-agent`): Requests OpenHands to consider only the issue/pull request description and **the specific comment**.
+
+## Advanced Settings
+
+### Add custom repository settings
+
+You can provide custom directions for OpenHands by following the [README for the resolver](https://github.com/All-Hands-AI/OpenHands/blob/main/openhands/resolver/README.md#providing-custom-instructions).
+
+### Configure custom macro
+
+To customize the default macro (`@openhands-agent`):
+
+1. [Create a repository variable](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/store-information-in-variables#creating-configuration-variables-for-a-repository) named `OPENHANDS_MACRO`
+2. Assign the variable a custom value
+
+## Writing Effective .openhands_instructions Files
+
+The `.openhands_instructions` file is a file that you can put in the root directory of your repository to guide OpenHands in understanding and working with your repository effectively. Here are key tips for writing high-quality instructions:
+
+### Core Principles
+
+1. **Concise but Informative**: Provide a clear, focused overview of the repository that emphasizes the most common actions OpenHands will need to perform.
+
+2. **Repository Structure**: Explain the key directories and their purposes, especially highlighting where different types of code (e.g., frontend, backend) are located.
+
+3. **Development Workflows**: Document the essential commands for:
+   - Building and setting up the project
+   - Running tests
+   - Linting and code quality checks
+   - Any environment-specific requirements
+
+4. **Testing Guidelines**: Specify:
+   - Where tests are located
+   - How to run specific test suites
+   - Any testing conventions or requirements
+
+### Example Structure
+
+```markdown
+# Repository Overview
+[Brief description of the project]
+
+## General Setup
+- Main build command
+- Development environment setup
+- Pre-commit checks
+
+## Backend
+- Location and structure
+- Testing instructions
+- Environment requirements
+
+## Frontend
+- Setup prerequisites
+- Build and test commands
+- Environment variables
+
+## Additional Guidelines
+- Code style requirements
+- Special considerations
+- Common workflows
+```
+
+For a real-world example, refer to the [OpenHands repository's .openhands_instructions](https://github.com/All-Hands-AI/OpenHands/blob/main/.openhands_instructions).