Fix issue #5219: Feature: PR Review

.github/workflows/openhands-resolver.yml

@@ -45,6 +45,83 @@ permissions:
   issues: write
 
 jobs:
+  review-pr:
+    if: |
+      github.event.label.name == 'review-pr'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.pull_request.head.sha }}
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install poetry
+        run: |
+          python -m pip install --upgrade pip
+          pip install poetry
+
+      - name: Install OpenHands from PR branch
+        run: |
+          poetry install
+
+      - name: Check required environment variables
+        env:
+          LLM_MODEL: ${{ secrets.LLM_MODEL }}
+          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
+          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
+          PAT_TOKEN: ${{ secrets.PAT_TOKEN }}
+          PAT_USERNAME: ${{ secrets.PAT_USERNAME }}
+        run: |
+          required_vars=("LLM_MODEL" "LLM_API_KEY" "PAT_TOKEN" "PAT_USERNAME")
+          for var in "${required_vars[@]}"; do
+            if [ -z "${!var}" ]; then
+              echo "Error: Required environment variable $var is not set."
+              exit 1
+            fi
+          done
+
+      - name: Set environment variables
+        run: |
+          echo "ISSUE_NUMBER=${{ github.event.pull_request.number }}" >> $GITHUB_ENV
+          echo "ISSUE_TYPE=pr" >> $GITHUB_ENV
+          echo "COMMENT_ID=None" >> $GITHUB_ENV
+          echo "MAX_ITERATIONS=1" >> $GITHUB_ENV
+          echo "SANDBOX_ENV_GITHUB_TOKEN=${{ secrets.GITHUB_TOKEN }}" >> $GITHUB_ENV
+          echo "TARGET_BRANCH=${{ inputs.target_branch }}" >> $GITHUB_ENV
+
+      - name: Comment on PR with start message
+        uses: actions/github-script@v7
+        with:
+          github-token: ${{secrets.GITHUB_TOKEN}}
+          script: |
+            github.rest.issues.createComment({
+              issue_number: ${{ env.ISSUE_NUMBER }},
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              body: `[OpenHands](https://github.com/All-Hands-AI/OpenHands) started reviewing the PR! You can monitor the progress [here](https://github.com/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}).`
+            });
+
+        run: |
+
+      - name: Review PR
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_USERNAME: ${{ secrets.PAT_USERNAME }}
+          LLM_MODEL: ${{ secrets.LLM_MODEL }}
+          LLM_API_KEY: ${{ secrets.LLM_API_KEY }}
+          LLM_BASE_URL: ${{ secrets.LLM_BASE_URL }}
+          PYTHONPATH: ""
+        run: |
+          cd /tmp && python -m openhands.resolver.review_pr \
+            --repo ${{ github.repository }} \
+            --issue-number ${{ env.ISSUE_NUMBER }} \
+            --issue-type ${{ env.ISSUE_TYPE }}
+
   auto-fix:
     if: |
       github.event_name == 'workflow_call' ||

openhands/resolver/prompts/review

@@ -0,0 +1,14 @@
+You are a helpful AI code reviewer. Your task is to review the following pull request:
+
+{{ body }}
+
+Please provide a thorough and constructive review that:
+1. Summarizes the changes and their purpose
+2. Evaluates code quality, readability, and maintainability
+3. Identifies potential bugs, edge cases, or performance issues
+4. Suggests improvements while being respectful and helpful
+5. Checks for test coverage and documentation
+6. Verifies the changes match the PR description and requirements
+
+Format your review with clear sections and use markdown for better readability.
+Focus on being specific and actionable in your feedback.

openhands/resolver/review_pr.py

@@ -0,0 +1,148 @@
+"""Module for reviewing pull requests."""
+
+import argparse
+import os
+import pathlib
+import subprocess
+from typing import Any
+
+import jinja2
+import litellm
+import requests
+
+from openhands.core.config import LLMConfig
+from openhands.core.logger import openhands_logger as logger
+from openhands.resolver.github_issue import GithubIssue
+from openhands.resolver.issue_definitions import PRHandler
+
+
+def get_pr_diff(owner: str, repo: str, pr_number: int, token: str) -> str:
+    """Get the diff for a pull request."""
+    url = f'https://api.github.com/repos/{owner}/{repo}/pulls/{pr_number}'
+    headers = {
+        'Authorization': f'token {token}',
+        'Accept': 'application/vnd.github.v3.diff',
+    }
+    response = requests.get(url, headers=headers)
+    response.raise_for_status()
+    return response.text
+
+
+def post_review_comment(owner: str, repo: str, pr_number: int, token: str, review: str) -> None:
+    """Post a review comment on a pull request."""
+    url = f'https://api.github.com/repos/{owner}/{repo}/issues/{pr_number}/comments'
+    headers = {
+        'Authorization': f'token {token}',
+        'Accept': 'application/vnd.github.v3+json',
+    }
+    data = {'body': review}
+    response = requests.post(url, headers=headers, json=data)
+    response.raise_for_status()
+
+
+def review_pr(
+    owner: str,
+    repo: str,
+    token: str,
+    username: str,
+    output_dir: str,
+    llm_config: LLMConfig,
+    issue_number: int,
+) -> None:
+    """Review a pull request.
+
+    Args:
+        owner: Github owner of the repo.
+        repo: Github repository name.
+        token: Github token to access the repository.
+        username: Github username to access the repository.
+        output_dir: Output directory to write the results.
+        llm_config: Configuration for the language model.
+        issue_number: PR number to review.
+    """
+    # Create output directory
+    pathlib.Path(output_dir).mkdir(parents=True, exist_ok=True)
+    pathlib.Path(os.path.join(output_dir, 'infer_logs')).mkdir(parents=True, exist_ok=True)
+    logger.info(f'Using output directory: {output_dir}')
+
+    # Get PR handler
+    pr_handler = PRHandler(owner, repo, token)
+
+    # Get PR details
+    issues: list[GithubIssue] = pr_handler.get_converted_issues(
+        issue_numbers=[issue_number], comment_id=None
+    )
+    pr = issues[0]
+
+    # Get PR diff
+    diff = get_pr_diff(owner, repo, issue_number, token)
+
+    # Load review template
+    with open(
+        os.path.join(os.path.dirname(__file__), 'prompts/review'),
+        'r',
+    ) as f:
+        template = jinja2.Template(f.read())
+
+    # Generate review instruction
+    instruction = template.render(
+        body=f'PR #{pr.number}: {pr.title}\n\n{pr.body}\n\nDiff:\n```diff\n{diff}\n```'
+    )
+
+    # Get review from LLM
+    response = litellm.completion(
+        model=llm_config.model,
+        messages=[{'role': 'user', 'content': instruction}],
+        api_key=llm_config.api_key,
+        base_url=llm_config.base_url,
+    )
+    review = response.choices[0].message.content.strip()
+
+    # Post review comment
+    post_review_comment(owner, repo, issue_number, token, review)
+    logger.info('Posted review comment successfully')
+
+
+def main() -> None:
+    """Main function."""
+    parser = argparse.ArgumentParser(description='Review a pull request.')
+    parser.add_argument('--repo', type=str, required=True, help='Repository in owner/repo format')
+    parser.add_argument('--issue-number', type=int, required=True, help='PR number to review')
+    parser.add_argument('--issue-type', type=str, required=True, help='Issue type (pr)')
+
+    args = parser.parse_args()
+
+    # Get environment variables
+    token = os.environ.get('GITHUB_TOKEN')
+    username = os.environ.get('GITHUB_USERNAME')
+    llm_model = os.environ.get('LLM_MODEL')
+    llm_api_key = os.environ.get('LLM_API_KEY')
+    llm_base_url = os.environ.get('LLM_BASE_URL')
+
+    if not all([token, username, llm_model, llm_api_key]):
+        raise ValueError('Missing required environment variables')
+
+    # Split repo into owner and name
+    owner, repo = args.repo.split('/')
+
+    # Configure LLM
+    llm_config = LLMConfig(
+        model=llm_model,
+        api_key=llm_api_key,
+        base_url=llm_base_url,
+    )
+
+    # Review PR
+    review_pr(
+        owner=owner,
+        repo=repo,
+        token=token,
+        username=username,
+        output_dir='/tmp/output',
+        llm_config=llm_config,
+        issue_number=args.issue_number,
+    )
+
+
+if __name__ == '__main__':
+    main()

tests/unit/test_review_pr.py

@@ -0,0 +1,96 @@
+"""Tests for the review_pr module."""
+
+import os
+import tempfile
+from unittest.mock import MagicMock, patch
+
+import pytest
+
+from openhands.core.config import LLMConfig
+from openhands.resolver.github_issue import GithubIssue
+from openhands.resolver.review_pr import get_pr_diff, post_review_comment, review_pr
+
+
+@pytest.fixture
+def mock_pr() -> GithubIssue:
+    """Create a mock PR."""
+    return GithubIssue(
+        owner='owner',
+        repo='repo',
+        number=1,
+        title='Test PR',
+        body='Test PR description',
+        thread_comments=None,
+        review_comments=None,
+    )
+
+
+@pytest.fixture
+def mock_llm_config() -> LLMConfig:
+    """Create a mock LLM config."""
+    return LLMConfig(
+        model='test-model',
+        api_key='test-key',
+        base_url=None,
+    )
+
+
+def test_get_pr_diff() -> None:
+    """Test getting PR diff."""
+    with patch('requests.get') as mock_get:
+        mock_get.return_value.text = 'test diff'
+        diff = get_pr_diff('owner', 'repo', 1, 'token')
+        assert diff == 'test diff'
+        mock_get.assert_called_once_with(
+            'https://api.github.com/repos/owner/repo/pulls/1',
+            headers={
+                'Authorization': 'token token',
+                'Accept': 'application/vnd.github.v3.diff',
+            },
+        )
+
+
+def test_post_review_comment() -> None:
+    """Test posting review comment."""
+    with patch('requests.post') as mock_post:
+        post_review_comment('owner', 'repo', 1, 'token', 'test review')
+        mock_post.assert_called_once_with(
+            'https://api.github.com/repos/owner/repo/issues/1/comments',
+            headers={
+                'Authorization': 'token token',
+                'Accept': 'application/vnd.github.v3+json',
+            },
+            json={'body': 'test review'},
+        )
+
+
+def test_review_pr(mock_pr: GithubIssue, mock_llm_config: LLMConfig) -> None:
+    """Test reviewing PR."""
+    with (
+        tempfile.TemporaryDirectory() as temp_dir,
+        patch('openhands.resolver.review_pr.PRHandler') as mock_handler,
+        patch('openhands.resolver.review_pr.get_pr_diff') as mock_get_diff,
+        patch('openhands.resolver.review_pr.post_review_comment') as mock_post_comment,
+        patch('litellm.completion') as mock_completion,
+    ):
+        # Setup mocks
+        mock_handler.return_value.get_converted_issues.return_value = [mock_pr]
+        mock_get_diff.return_value = 'test diff'
+        mock_completion.return_value.choices = [MagicMock(message=MagicMock(content='test review'))]
+
+        # Run review
+        review_pr(
+            owner='owner',
+            repo='repo',
+            token='token',
+            username='username',
+            output_dir=temp_dir,
+            llm_config=mock_llm_config,
+            issue_number=1,
+        )
+
+        # Verify calls
+        mock_handler.assert_called_once_with('owner', 'repo', 'token')
+        mock_get_diff.assert_called_once_with('owner', 'repo', 1, 'token')
+        mock_completion.assert_called_once()
+        mock_post_comment.assert_called_once_with('owner', 'repo', 1, 'token', 'test review')