Skip to content

Commit

Permalink
Merge branch 'main' into feat-extra-security-host
Browse files Browse the repository at this point in the history
  • Loading branch information
@tofarr
tofarr authored Dec 28, 2024
2 parents 3e4fef6 + ebb2d86 commit 7a9d4ab
Show file tree
Hide file tree
Showing 2 changed files with 323 additions and 2 deletions.
316 changes: 316 additions & 0 deletions evaluation/benchmarks/the_agent_company/scripts/summarise_results.py
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,316 @@
###########################################################################################################
# Adapted from https://github.com/TheAgentCompany/TheAgentCompany/blob/main/evaluation/summarise_results.py
###########################################################################################################


import glob
import json
import os
import re
import sys
from typing import Dict, Tuple


def calculate_cost(model: str, prompt_tokens: int, completion_tokens: int) -> float:
"""
Calculate the cost of the model call.
"""
if 'claude-3-5-sonnet' in model.lower():
# https://www.anthropic.com/pricing#anthropic-api, accessed 12/11/2024
return 0.000003 * prompt_tokens + 0.000015 * completion_tokens
elif 'gpt-4o' in model.lower():
# https://openai.com/api/pricing/, accessed 12/11/2024
return 0.0000025 * prompt_tokens + 0.00001 * completion_tokens
elif 'gemini-1.5-pro' in model.lower():
# https://ai.google.dev/pricing#1_5pro, accessed 12/11/2024
# assuming prompts up to 128k tokens
cost = 0.00000125 * prompt_tokens + 0.000005 * completion_tokens
if prompt_tokens > 128000:
cost *= 2
return cost
elif 'gemini-2.0-flash-exp' in model.lower():
# price unknown for gemini-2.0-flash-exp, assuming same price as gemini-1.5-flash
cost = 0.000000075 * prompt_tokens + 0.0000003 * completion_tokens
if prompt_tokens > 128000:
cost *= 2
return cost
elif 'qwen2-72b' in model.lower():
# assuming hosted on Together
# https://www.together.ai/pricing, accessed 12/11/2024
return 0.0000009 * (prompt_tokens + completion_tokens)
elif 'qwen2p5-72b' in model.lower():
# assuming hosted on Together
# https://www.together.ai/pricing, accessed 12/14/2024
return 0.0000012 * (prompt_tokens + completion_tokens)
elif 'llama-v3p1-405b-instruct' in model.lower():
# assuming hosted on Fireworks AI
# https://fireworks.ai/pricing, accessed 12/11/2024
return 0.000003 * (prompt_tokens + completion_tokens)
elif 'llama-v3p1-70b-instruct' in model.lower():
# assuming hosted on Fireworks AI
return 0.0000009 * (prompt_tokens + completion_tokens)
elif 'llama-v3p3-70b-instruct' in model.lower():
# assuming hosted on Fireworks AI
return 0.0000009 * (prompt_tokens + completion_tokens)
elif 'amazon.nova-pro-v1:0' in model.lower():
# assuming hosted on Amazon Bedrock
# https://aws.amazon.com/bedrock/pricing/, accessed 12/11/2024
return 0.0000008 * prompt_tokens + 0.0000032 * completion_tokens
else:
raise ValueError(f'Unknown model: {model}')


def analyze_eval_json_file(filepath: str) -> Tuple[int, int]:
"""
Analyze a single eval JSON file and extract the total and result from final_score.
Args:
filepath: Path to the JSON file
Returns:
Tuple containing (total, result) from final_score
"""
try:
with open(filepath, 'r') as f:
data = json.load(f)

final_score = data.get('final_score', {})
return (final_score.get('total', 0), final_score.get('result', 0))
except json.JSONDecodeError as e:
print(f'Error decoding JSON in {filepath}: {e}')
return (0, 0)
except Exception as e:
print(f'Error processing {filepath}: {e}')
return (0, 0)


def analyze_traj_json_file(filepath: str) -> Tuple[int, float]:
"""
Analyze a single trajectory JSON file and extract the steps and tokens
for each step. Then estimate the cost based on the tokens and the model type.
Note: this is assuming there's no prompt caching at all.
"""
steps: int = 0
cost: float = 0.0
with open(filepath, 'r') as f:
data = json.load(f)
response_id = None
for action in data:
if 'tool_call_metadata' in action:
if action['tool_call_metadata']['model_response']['id'] != response_id:
response_id = action['tool_call_metadata']['model_response']['id']
else:
# openhands displays the same model response meta data multiple times, when
# a single LLM call leads to multiple actions and observations.
continue
steps += 1
usage = action['tool_call_metadata']['model_response']['usage']
model: str = action['tool_call_metadata']['model_response']['model']
prompt_tokens = usage['prompt_tokens']
completion_tokens = usage['completion_tokens']
cost += calculate_cost(model, prompt_tokens, completion_tokens)

return (steps, cost)


def analyze_folder(
folder_path: str,
) -> Tuple[Dict[str, Tuple[int, int]], Dict[str, Tuple[int, float]]]:
"""
Analyze all eval_*.json & traj_*.json files in the specified folder.
Args:
folder_path: Path to the folder containing JSON files
Returns:
dictionaries:
- eval_results: Dictionary with filename as key and (total, result) tuple as value
- traj_results: Dictionary with filename as key and (steps, cost) tuple as value
"""
eval_results = {}
traj_results = {}

eval_pattern = os.path.join(folder_path, 'eval_*.json')
traj_pattern = os.path.join(folder_path, 'traj_*.json')

for filepath in glob.glob(eval_pattern):
filename = os.path.basename(filepath)
total, result = analyze_eval_json_file(filepath)
key = re.search(r'eval_(.+)\.json', filename).group(1)
eval_results[key] = (total, result)

for filepath in glob.glob(traj_pattern):
filename = os.path.basename(filepath)
steps, cost = analyze_traj_json_file(filepath)
key = re.search(r'traj_(.+)\.json', filename).group(1)
traj_results[key] = (steps, cost)

return eval_results, traj_results


def get_task_nature_category(task_name: str) -> str:
"""
Get the nature category of the task.
"""
task_nature = task_name.split('-')[0]
if task_nature.lower() in ['sde', 'pm', 'ds', 'admin', 'hr', 'finance']:
return task_nature
else:
return 'other'


def calculate_score(total: int, result: int) -> float:
"""
Calculate the score as a number between 0 and 1.
Formula: score = (result / total) * 0.5 + (result // total) * 0.5
Explanation:
- (result / total) * 0.5: This is the completion ratio, scaled down to a 0-0.5 range.
- (result // total) * 0.5: This is a binary score indicating whether the task was completed or not.
Args:
total: Total possible points
result: Actual points achieved
Returns:
Score as a number between 0 and 1
"""
return (result / total * 0.5) + (result // total * 0.5)


def is_perfect_completion(total: int, result: int) -> bool:
"""
Check if the task achieved perfect completion.
Args:
total: Total possible points
result: Actual points achieved
Returns:
True if result equals total, False otherwise
"""
return total > 0 and total == result


def main():
if len(sys.argv) != 2:
print('Usage: poetry run python summarise_results.py <folder_path>')
sys.exit(1)

folder_path = sys.argv[1]

if not os.path.isdir(folder_path):
print(f"Error: '{folder_path}' is not a valid directory")
sys.exit(1)

eval_results, traj_results = analyze_folder(folder_path)

if not eval_results:
print(f'No eval_*.json files found in {folder_path}')
return

# Create list of results with completion ratios for sorting
detailed_results = [
(
task_name,
total,
result,
calculate_score(total, result),
is_perfect_completion(total, result),
get_task_nature_category(task_name),
)
for task_name, (total, result) in eval_results.items()
]

# Sort by score in descending order
detailed_results.sort(key=lambda x: (-x[3], x[0]))

# Calculate perfect completion stats
perfect_completions = sum(
1 for _, _, _, _, is_perfect, _ in detailed_results if is_perfect
)

# Print header
print('\n# Evaluation Results Report')
print('\n## Results per File')
print('\n*Sorted by score (⭐ indicates perfect completion)*\n')

# Print table header
print(
'| Filename | Total | Result | Score | Steps | Cost (assuming no prompt caching)|'
)
print('|----------|--------|---------|-------|-------|------|')

# Print individual file results
for task_name, total, result, score, is_perfect, task_nature in detailed_results:
perfect_marker = ' ⭐' if is_perfect else ''
print(
f'| {task_name} | {total:,} | {result:,} | {score:.2f}{perfect_marker} | {traj_results[task_name][0]} | {traj_results[task_name][1]:.2f} |'
)

# Print summary section
print('\n## Summary\n')
print(f'**Tasks Evaluated:** {len(eval_results)}\n')
print(
f'**Perfect Completions:** {perfect_completions}/{len(eval_results)} ({(perfect_completions/len(eval_results)*100):.2f}%)\n'
)

overall_score = (
sum(score for _, _, _, score, _, _ in detailed_results)
/ len(detailed_results)
* 100
)
avg_steps = sum(steps for steps, _ in traj_results.values()) / len(traj_results)
avg_cost = sum(cost for _, cost in traj_results.values()) / len(traj_results)
print(f'**Overall Score:** {overall_score:.2f}%\n')
print(f'**Average Steps:** {avg_steps:.2f}\n')
print(f'**Average Cost (USD):** {avg_cost:.2f}\n')

# Additional statistics
if detailed_results:
highest_score = max(score for _, _, _, score, _, _ in detailed_results)
lowest_score = min(score for _, _, _, score, _, _ in detailed_results)
median_score = detailed_results[len(detailed_results) // 2][3]
avg_score = sum(score for _, _, _, score, _, _ in detailed_results) / len(
detailed_results
)

print('\n## Statistics\n')
print('| Metric | Value |')
print('|---------|--------|')
print(f'| Highest Task Score | {highest_score*100:.2f}% |')
print(f'| Lowest Task Score | {lowest_score*100:.2f}% |')
print(f'| Median Task Score | {median_score*100:.2f}% |')
print(f'| Average Task Score | {avg_score*100:.2f}% |')

# compute avg score per nature category
print('\n## Statistics per Nature Category\n')
print('| Metric | Value |')
print('|---------|--------|')
for task_nature in ['sde', 'pm', 'ds', 'admin', 'hr', 'finance', 'other']:
num_of_tasks = sum(
1
for _, _, _, _, _, nature_category in detailed_results
if nature_category == task_nature
)
task_nature_score = (
sum(
score
for _, _, _, score, _, nature_category in detailed_results
if nature_category == task_nature
)
/ num_of_tasks
)
perfect_completions = sum(
1
for _, _, _, _, is_perfect, nature_category in detailed_results
if nature_category == task_nature and is_perfect
)
print(
f'| Perfect Completions for {task_nature} | {perfect_completions}/{num_of_tasks} ({perfect_completions/num_of_tasks*100:.2f}%) |'
)
print(f'| Average Score for {task_nature} | {task_nature_score*100:.2f}% |')


if __name__ == '__main__':
main()
9 changes: 7 additions & 2 deletions openhands/core/main.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,9 +249,14 @@ def auto_continue_response(
try_parse: Callable[[Action | None], str] | None = None,
) -> str:
"""Default function to generate user responses.
Returns 'continue' to tell the agent to proceed without asking for more input.
Tell the agent to proceed without asking for more input, or finish the interaction.
"""
return 'continue'
message = (
'Please continue on whatever approach you think is suitable.\n'
'If you think you have solved the task, please finish the interaction.\n'
'IMPORTANT: YOU SHOULD NEVER ASK FOR HUMAN RESPONSE.\n'
)
return message


if __name__ == '__main__':
Expand Down

0 comments on commit 7a9d4ab

Please sign in to comment.