Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Use keyword matching for CodeAct microagents #4568

Merged
merged 34 commits into from
Nov 9, 2024
Merged
Show file tree
Hide file tree
Changes from 6 commits
Commits
Show all changes
34 commits
Select commit Hold shift + click to select a range
e798bc6
first pass at gh microagent triggers
rbren Oct 16, 2024
c9a3cc7
first pass at using gh micro
rbren Oct 17, 2024
2461491
Merge branch 'main' into rb/gh-micro-agent
rbren Oct 21, 2024
186f2ac
more instructions
rbren Oct 21, 2024
f45fa1c
Merge branch 'main' into rb/gh-micro-agent
rbren Oct 25, 2024
56a9469
Update frontend/src/components/project-menu/ProjectMenuCard.tsx
rbren Oct 25, 2024
2669fdb
fix test
rbren Oct 25, 2024
004ffc0
Merge branch 'main' into rb/gh-micro-agent
rbren Oct 25, 2024
d891626
fix tests
rbren Oct 25, 2024
32b7ef2
better messages
rbren Oct 25, 2024
4f5c8f9
better prompt hints
rbren Oct 25, 2024
f7ee9fe
more fixes
rbren Oct 25, 2024
eb0f056
fix up last_user_message logic
rbren Oct 25, 2024
3366e87
move env reminder back to bottom
rbren Oct 25, 2024
300c0fc
remove microagents template
rbren Oct 25, 2024
38006d8
Merge branch 'main' into rb/gh-micro-agent
rbren Nov 7, 2024
f450d74
fix some merge issues
rbren Nov 7, 2024
19782be
fix some merge issues
rbren Nov 7, 2024
c087dc8
fix function calling prompt
rbren Nov 7, 2024
f8a1c35
remove extra promptman
rbren Nov 7, 2024
d847a54
fix dirs
rbren Nov 7, 2024
0724514
make easter egg less likely to cause problems
rbren Nov 7, 2024
64a1c61
Merge branch 'main' into rb/gh-micro-agent
rbren Nov 7, 2024
b7889bd
fix tets
rbren Nov 7, 2024
44d27bf
lint
rbren Nov 7, 2024
8d772eb
fix tests
rbren Nov 7, 2024
efdd230
use xml
rbren Nov 7, 2024
0cae5ec
always decorate messages to fix caching
rbren Nov 7, 2024
e548756
lint
rbren Nov 7, 2024
64627e3
update eval for api change
rbren Nov 8, 2024
199ee9e
Update openhands/agenthub/codeact_agent/micro/github.md
rbren Nov 8, 2024
cb5614e
Update openhands/agenthub/codeact_agent/micro/github.md
rbren Nov 8, 2024
c016264
Merge branch 'main' into rb/gh-micro-agent
rbren Nov 8, 2024
b8879e7
Update openhands/agenthub/codeact_agent/micro/github.md
rbren Nov 8, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 1 addition & 4 deletions frontend/src/components/project-menu/ProjectMenuCard.tsx
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,7 @@ export function ProjectMenuCard({
const handlePushToGitHub = () => {
const rawEvent = {
content: `
Let's push the code to GitHub.
If we're currently on the openhands-workspace branch, please create a new branch with a descriptive name.
Commit any changes and push them to the remote repository.
Finally, open up a pull request using the GitHub API and the token in the GITHUB_TOKEN environment variable, then show me the URL of the pull request.
Please push the changes to GitHub and open a pull request.
`,
imageUrls: [],
timestamp: new Date().toISOString(),
Expand Down
81 changes: 36 additions & 45 deletions openhands/agenthub/codeact_agent/codeact_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
JupyterRequirement,
PluginRequirement,
)
from openhands.utils.microagent import MicroAgent
from openhands.utils.prompt import PromptManager


Expand Down Expand Up @@ -79,20 +78,9 @@ def __init__(
super().__init__(llm, config)
self.reset()

self.micro_agent = (
MicroAgent(
os.path.join(
os.path.dirname(__file__), 'micro', f'{config.micro_agent_name}.md'
)
)
if config.micro_agent_name
else None
)

self.prompt_manager = PromptManager(
prompt_dir=os.path.join(os.path.dirname(__file__)),
agent_skills_docs=AgentSkillsRequirement.documentation,
micro_agent=self.micro_agent,
)

def action_to_str(self, action: Action) -> str:
Expand Down Expand Up @@ -222,26 +210,7 @@ def step(self, state: State) -> Action:
return self.action_parser.parse(response)

def _get_messages(self, state: State) -> list[Message]:
messages: list[Message] = [
Message(
role='system',
content=[
TextContent(
text=self.prompt_manager.system_message,
cache_prompt=self.llm.is_caching_prompt_active(), # Cache system prompt
)
],
),
Message(
role='user',
content=[
TextContent(
text=self.prompt_manager.initial_user_message,
cache_prompt=self.llm.is_caching_prompt_active(), # if the user asks the same query,
)
],
),
]
messages: list[Message] = []

for event in state.history.get_events():
# create a regular message from an event
Expand All @@ -252,7 +221,6 @@ def _get_messages(self, state: State) -> list[Message]:
else:
raise ValueError(f'Unknown event type: {type(event)}')

# add regular message
if message:
# handle error if the message is the SAME role as the previous message
# litellm.exceptions.BadRequestError: litellm.BadRequestError: OpenAIException - Error code: 400 - {'detail': 'Only supports u/a/u/a/u...'}
Expand All @@ -262,18 +230,6 @@ def _get_messages(self, state: State) -> list[Message]:
else:
messages.append(message)

# Add caching to the last 2 user messages
if self.llm.is_caching_prompt_active():
user_turns_processed = 0
for message in reversed(messages):
if message.role == 'user' and user_turns_processed < 2:
message.content[
-1
].cache_prompt = True # Last item inside the message content
user_turns_processed += 1

# The latest user message is important:
# we want to remind the agent of the environment constraints
latest_user_message = next(
islice(
(
Expand All @@ -286,8 +242,43 @@ def _get_messages(self, state: State) -> list[Message]:
),
None,
)
messages = (
[
Message(
role='system',
content=[
TextContent(
text=self.prompt_manager.get_system_message(),
cache_prompt=self.llm.is_caching_prompt_active(), # Cache system prompt
)
],
),
Message(
role='user',
content=[
TextContent(
text=self.prompt_manager.get_example_user_message(
''
if latest_user_message is None
else latest_user_message.content[-1].text
),
cache_prompt=self.llm.is_caching_prompt_active(), # if the user asks the same query,
)
],
),
]
+ messages
)

if latest_user_message:
reminder_text = f'\n\nENVIRONMENT REMINDER: You have {state.max_iterations - state.iteration} turns left to complete the task. When finished reply with <finish></finish>.'
latest_user_message.content.append(TextContent(text=reminder_text))

if self.llm.is_caching_prompt_active():
user_turns_processed = 0
for message in reversed(messages):
if message.role == 'user' and user_turns_processed < 2:
message.content[-1].cache_prompt = True
user_turns_processed += 1

return messages
90 changes: 27 additions & 63 deletions openhands/agenthub/codeact_agent/micro/github.md
Original file line number Diff line number Diff line change
@@ -1,69 +1,33 @@
---
name: github
agent: CodeActAgent
require_env_var:
SANDBOX_ENV_GITHUB_TOKEN: "Create a GitHub Personal Access Token (https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens) and set it as SANDBOX_GITHUB_TOKEN in your environment variables."
triggers:
- github
- git
---

# How to Interact with Github

## Environment Variable Available

- `GITHUB_TOKEN`: A read-only token for Github.

## Using GitHub's RESTful API

Use `curl` with the `GITHUB_TOKEN` to interact with GitHub's API. Here are some common operations:

Here's a template for API calls:

```sh
curl -H "Authorization: token $GITHUB_TOKEN" \
"https://api.github.com/{endpoint}"
You have access to an environment variable, `GITHUB_TOKEN`, which allows you to interact with
the GitHub API.

You can use `curl` with the `GITHUB_TOKEN` to interact with GitHub's API.
ALWAYS use the GitHub API for operations instead of a web browser.

Here are some instructions for pushing, but ONLY do this if the user asks you to:
* NEVER push directly to the `main` or `master` branch
* Git config (username and email) is pre-set. Do not modify.
* You may already be on a branch called `openhands-workspace`. Create a new branch with a better name before pushing.
* Use the GitHub API to create a pull request, if you haven't already
* Use the main branch as the base branch, unless the user requests otherwise
* After opening or updating a pull request, send the user a short message with a link to the pull request.
* Do all of the above in as few steps as possible. E.g. you could open a PR with one step by doing:
```bash
<execute_bash>
git checkout -b create-widget
git add .
git commit -m "Create widget"
git push origin create-widget
curl -X POST "https://api.github.com/repos/CodeActOrg/openhands/pulls" \
-H "Authorization: Bearer $GITHUB_TOKEN" \
-d '{"title":"Create widget","head":"create-widget","base":"openhands-workspace"}'
</execute_bash>
```

First replace `{endpoint}` with the specific API path. Common operations:

1. View an issue or pull request:
- Issues: `/repos/{owner}/{repo}/issues/{issue_number}`
- Pull requests: `/repos/{owner}/{repo}/pulls/{pull_request_number}`

2. List repository issues or pull requests:
- Issues: `/repos/{owner}/{repo}/issues`
- Pull requests: `/repos/{owner}/{repo}/pulls`

3. Search issues or pull requests:
- `/search/issues?q=repo:{owner}/{repo}+is:{type}+{search_term}+state:{state}`
- Replace `{type}` with `issue` or `pr`

4. List repository branches:
`/repos/{owner}/{repo}/branches`

5. Get commit details:
`/repos/{owner}/{repo}/commits/{commit_sha}`

6. Get repository details:
`/repos/{owner}/{repo}`

7. Get user information:
`/user`

8. Search repositories:
`/search/repositories?q={query}`

9. Get rate limit status:
`/rate_limit`

Replace `{owner}`, `{repo}`, `{commit_sha}`, `{issue_number}`, `{pull_request_number}`,
`{search_term}`, `{state}`, and `{query}` with appropriate values.

## Important Notes

1. Always use the GitHub API for operations instead of a web browser.
2. The `GITHUB_TOKEN` is read-only. Avoid operations that require write access.
3. Git config (username and email) is pre-set. Do not modify.
4. Edit and test code locally. Never push directly to remote.
5. Verify correct branch before committing.
6. Commit changes frequently.
7. If the issue or task is ambiguous or lacks sufficient detail, always request clarification from the user before proceeding.
8. You should avoid using command line tools like `sed` for file editing.
6 changes: 4 additions & 2 deletions openhands/agenthub/codeact_agent/user_prompt.j2
Original file line number Diff line number Diff line change
Expand Up @@ -215,11 +215,13 @@ The server is running on port 5000 with PID 126. You can access the list of numb
{% endset %}
Here is an example of how you can interact with the environment for task solving:
{{ DEFAULT_EXAMPLE }}
{% if micro_agent %}

{% if micro_agents|length > 0 %}
--- BEGIN OF GUIDELINE ---
The following information may assist you in completing your task:

{% for micro_agent in micro_agents %}
{{ micro_agent }}
{% endfor %}
--- END OF GUIDELINE ---
{% endif %}

Expand Down
44 changes: 25 additions & 19 deletions openhands/utils/microagent.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,15 +3,11 @@
import frontmatter
import pydantic

from openhands.controller.agent import Agent
from openhands.core.exceptions import MicroAgentValidationError
from openhands.core.logger import openhands_logger as logger


class MicroAgentMetadata(pydantic.BaseModel):
name: str
agent: str
require_env_var: dict[str, str]
triggers: list[str] = []


class MicroAgent:
Expand All @@ -23,22 +19,32 @@ def __init__(self, path: str):
self._loaded = frontmatter.load(file)
self._content = self._loaded.content
self._metadata = MicroAgentMetadata(**self._loaded.metadata)
self._validate_micro_agent()

def should_trigger(self, message: str) -> bool:
message = message.lower()
for trigger in self.triggers:
if trigger.lower() in message:
print(f'Triggered {self.name} with {trigger}')
print(f'Message: {message}')
return True
return False

@property
def content(self) -> str:
return self._content

def _validate_micro_agent(self):
logger.info(
f'Loading and validating micro agent [{self._metadata.name}] based on [{self._metadata.agent}]'
)
# Make sure the agent is registered
agent_cls = Agent.get_cls(self._metadata.agent)
assert agent_cls is not None
# Make sure the environment variables are set
for env_var, instruction in self._metadata.require_env_var.items():
if env_var not in os.environ:
raise MicroAgentValidationError(
f'Environment variable [{env_var}] is required by micro agent [{self._metadata.name}] but not set. {instruction}'
)
@property
def metadata(self) -> MicroAgentMetadata:
return self._metadata

@property
def name(self) -> str:
return self._metadata.name

@property
def triggers(self) -> list[str]:
return self._metadata.triggers

@property
def agent(self) -> str:
return self._metadata.agent
26 changes: 18 additions & 8 deletions openhands/utils/prompt.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,21 +16,29 @@ class PromptManager:
Attributes:
prompt_dir (str): Directory containing prompt templates.
agent_skills_docs (str): Documentation of agent skills.
micro_agent (MicroAgent | None): Micro-agent, if specified.
"""

def __init__(
self,
prompt_dir: str,
agent_skills_docs: str,
micro_agent: MicroAgent | None = None,
):
self.prompt_dir: str = prompt_dir
self.agent_skills_docs: str = agent_skills_docs

self.system_template: Template = self._load_template('system_prompt')
self.user_template: Template = self._load_template('user_prompt')
self.micro_agent: MicroAgent | None = micro_agent
self.microagents: dict = {}

micro_agent_dir = os.path.join(prompt_dir, 'micro')
micro_agent_files = [
os.path.join(micro_agent_dir, f)
for f in os.listdir(micro_agent_dir)
if f.endswith('.md')
]
for micro_agent_file in micro_agent_files:
micro_agent = MicroAgent(micro_agent_file)
self.microagents[micro_agent.name] = micro_agent

def _load_template(self, template_name: str) -> Template:
template_path = os.path.join(self.prompt_dir, f'{template_name}.j2')
Expand All @@ -39,15 +47,13 @@ def _load_template(self, template_name: str) -> Template:
with open(template_path, 'r') as file:
return Template(file.read())

@property
def system_message(self) -> str:
def get_system_message(self) -> str:
rendered = self.system_template.render(
agent_skills_docs=self.agent_skills_docs,
).strip()
return rendered

@property
def initial_user_message(self) -> str:
def get_example_user_message(self, latest_user_message: str) -> str:
"""This is the initial user message provided to the agent
before *actual* user instructions are provided.

Expand All @@ -57,7 +63,11 @@ def initial_user_message(self) -> str:
These additional context will convert the current generic agent
into a more specialized agent that is tailored to the user's task.
"""
micro_agent_prompts = []
for micro_agent in self.microagents.values():
if micro_agent.should_trigger(latest_user_message):
micro_agent_prompts.append(micro_agent.content)
rendered = self.user_template.render(
micro_agent=self.micro_agent.content if self.micro_agent else None
micro_agents=micro_agent_prompts,
)
return rendered.strip()