AI Agent Evaluation Planner

# Hermes Dispatch Tool — AI Agent Evaluation Planner
# Source: https://hermesdispatch.dev/tools/agent-evaluation-planner/
# Description: Choose the right benchmark, budget, and timeline for evaluating an AI agent.
# License: MIT (generated by hermesdispatch.dev)
#
# INSTALL:
#   1. Save this file as ~/.hermes/hermes-agent/tools/agent_evaluation_planner.py
#   2. Restart Hermes or run /reset in a session
#   3. The tool auto-registers if Hermes uses auto-discovery of tools/*.py
#
# MANUAL REGISTRY (if auto-discovery is off):
#   from tools.agent_evaluation_planner import register
#   register()

import json

DATA = {"last_updated": "2026-07-04", "note": "Effort, cost, and time estimates are directional. Real numbers depend on model choice, agent retries, infrastructure, and benchmark version. Use this planner to compare evaluation options before committing resources.", "agent_types": [{"slug": "coding", "name": "Coding / software engineering", "description": "Agents that write, edit, debug, and ship code in repositories."}, {"slug": "general", "name": "General assistant / copilot", "description": "Multi-purpose agents that answer questions, browse, and use tools."}, {"slug": "web", "name": "Web / browser automation", "description": "Agents that navigate websites, fill forms, and complete browser tasks."}, {"slug": "desktop", "name": "Desktop / OS automation", "description": "Agents that control apps, files, and OS-level workflows."}, {"slug": "multi", "name": "Multi-agent / orchestration", "description": "Teams of agents that delegate, coordinate, and share state."}, {"slug": "voice", "name": "Voice / conversational", "description": "Agents that handle real-time speech, turn-taking, and call flows."}, {"slug": "domain", "name": "Domain-specific (math, science, security, finance)", "description": "Agents tuned for a specialized field with narrow, high-accuracy tasks."}], "readiness_levels": [{"slug": "prototype", "name": "Prototype / demo", "description": "Need a quick signal that the agent can do the task at all."}, {"slug": "iteration", "name": "Iteration / regression testing", "description": "Need fast, cheap feedback to compare prompts, models, or tool changes."}, {"slug": "release", "name": "Pre-release validation", "description": "Need a trustworthy score to decide if the agent is ready for users."}, {"slug": "leaderboard", "name": "Leaderboard / public benchmark", "description": "Need a credible number for marketing, investors, or research publication."}], "budgets": [{"slug": "tiny", "name": "<$50/run", "max": 50}, {"slug": "small", "name": "$50-250/run", "max": 250}, {"slug": "medium", "name": "$250-1k/run", "max": 1000}, {"slug": "large", "name": "$1k-5k/run", "max": 5000}, {"slug": "unlimited", "name": "$5k+/run", "max": 100000}], "benchmarks": [{"slug": "swe-bench", "name": "SWE-bench", "category": "coding", "what": "Fix real GitHub issues end-to-end in popular Python repositories.", "effort": "High", "time_hours": 24, "usd_per_task": 25, "task_count": 2294, "best_for": ["coding"], "readiness": ["release", "leaderboard"], "budget": ["medium", "large", "unlimited"], "pros": "Industry standard for real-world coding agents; tests repo-level reasoning.", "cons": "Expensive; slow; public issues can lead to overfitting.", "link": "https://www.swebench.com/"}, {"slug": "swe-bench-verified", "name": "SWE-bench Verified", "category": "coding", "what": "Curated 500-instance subset with confirmed problem/solution pairs.", "effort": "High", "time_hours": 12, "usd_per_task": 20, "task_count": 500, "best_for": ["coding"], "readiness": ["release", "leaderboard"], "budget": ["small", "medium", "large", "unlimited"], "pros": "More reliable than full SWE-bench; good for vendor comparison.", "cons": "Still costly; may not match your internal codebase.", "link": "https://www.swebench.com/"}, {"slug": "swe-bench-lite", "name": "SWE-bench Lite", "category": "coding", "what": "Smaller, simpler subset designed for faster iteration.", "effort": "Medium", "time_hours": 4, "usd_per_task": 5, "task_count": 300, "best_for": ["coding"], "readiness": ["prototype", "iteration"], "budget": ["tiny", "small", "medium"], "pros": "Cheaper and faster than full SWE-bench; good CI signal.", "cons": "Easier; may not generalize to full production issues.", "link": "https://www.swebench.com/"}, {"slug": "humaneval", "name": "HumanEval", "category": "coding", "what": "164 hand-written function-level programming problems with unit tests.", "effort": "Low", "time_hours": 1, "usd_per_task": 0.5, "task_count": 164, "best_for": ["coding"], "readiness": ["prototype", "iteration"], "budget": ["tiny"], "pros": "Fast, cheap, reproducible; great for model comparison.", "cons": "Toy problems; no repo-level reasoning.", "link": "https://github.com/openai/human-eval"}, {"slug": "mbpp", "name": "MBPP", "category": "coding", "what": "Crowd-sourced Python programming problems.", "effort": "Low", "time_hours": 1, "usd_per_task": 0.3, "task_count": 974, "best_for": ["coding"], "readiness": ["prototype", "iteration"], "budget": ["tiny", "small"], "pros": "Larger and more varied than HumanEval.", "cons": "Simpler than real-world coding; limited scope.", "link": "https://github.com/google-research/google-research/tree/master/mbpp"}, {"slug": "gaia", "name": "GAIA", "category": "general", "what": "466 real-world assistant tasks requiring reasoning, browsing, and tool use.", "effort": "High", "time_hours": 16, "usd_per_task": 1.5, "task_count": 466, "best_for": ["general"], "readiness": ["release", "leaderboard"], "budget": ["small", "medium", "large"], "pros": "Strong signal for general assistant ability; human-aligned difficulty.", "cons": "Some tasks need live tools; leaderboard costs add up.", "link": "https://huggingface.co/datasets/maismoo/GAIA"}, {"slug": "mmlu", "name": "MMLU / MMLU-Pro", "category": "general", "what": "Multiple-choice questions spanning STEM, humanities, and social sciences.", "effort": "Low", "time_hours": 1, "usd_per_task": 0.05, "task_count": 12000, "best_for": ["general", "domain"], "readiness": ["prototype", "iteration"], "budget": ["tiny"], "pros": "Standardized; easy to run; good for model ranking.", "cons": "Static knowledge, not agent behavior.", "link": "https://github.com/simpler-conversation/MMLU"}, {"slug": "livebench", "name": "LiveBench", "category": "general", "what": "Questions with answers that change over time to reduce memorization.", "effort": "Low", "time_hours": 2, "usd_per_task": 0.05, "task_count": 1000, "best_for": ["general"], "readiness": ["iteration", "release"], "budget": ["tiny", "small"], "pros": "Harder to game; tests current knowledge.", "cons": "Limited coverage; not a full agent test.", "link": "https://livebench.ai/"}, {"slug": "webarena", "name": "WebArena", "category": "web", "what": "812 reproducible tasks across self-hosted web apps.", "effort": "High", "time_hours": 20, "usd_per_task": 1.5, "task_count": 812, "best_for": ["web"], "readiness": ["release", "leaderboard"], "budget": ["small", "medium", "large"], "pros": "Controlled environment; tests realistic web interaction.", "cons": "Setup heavy; simulated sites differ from live web.", "link": "https://webarena.dev/"}, {"slug": "webvoyager", "name": "WebVoyager", "category": "web", "what": "Tasks on live websites using screenshots and actions.", "effort": "High", "time_hours": 24, "usd_per_task": 4, "task_count": 643, "best_for": ["web"], "readiness": ["release", "leaderboard"], "budget": ["medium", "large", "unlimited"], "pros": "Tests real-world resilience: layout changes, captchas, JS.", "cons": "Noisy; slow; live sites can break evals.", "link": "https://github.com/MinorJerry/WebVoyager"}, {"slug": "mind2web", "name": "Mind2Web", "category": "web", "what": "Diverse web tasks across 137 sites in 31 domains.", "effort": "Medium", "time_hours": 8, "usd_per_task": 1, "task_count": 2350, "best_for": ["web"], "readiness": ["prototype", "iteration", "release"], "budget": ["small", "medium"], "pros": "Broad domain coverage; good cross-site generalization test.", "cons": "Annotation-based; less end-to-end than WebArena.", "link": "https://osu-nlp-group.github.io/Mind2Web/"}, {"slug": "osworld", "name": "OSWorld", "category": "desktop", "what": "369 tasks across Windows, Ubuntu, and macOS requiring GUI + API reasoning.", "effort": "Very High", "time_hours": 40, "usd_per_task": 5, "task_count": 369, "best_for": ["desktop"], "readiness": ["release", "leaderboard"], "budget": ["large", "unlimited"], "pros": "Tests real desktop automation; multimodal.", "cons": "Requires VM infrastructure; slow to evaluate.", "link": "https://osworld-public.github.io/"}, {"slug": "windows-agent-arena", "name": "Windows Agent Arena", "category": "desktop", "what": "154 tasks across Windows applications and system settings.", "effort": "Very High", "time_hours": 32, "usd_per_task": 6, "task_count": 154, "best_for": ["desktop"], "readiness": ["release", "leaderboard"], "budget": ["large", "unlimited"], "pros": "Focused on Windows enterprise automation scenarios.", "cons": "Windows-specific; smaller coverage than OSWorld.", "link": "https://microsoft.github.io/WindowsAgentArena/"}, {"slug": "agentbench", "name": "AgentBench", "category": "multi", "what": "Eight environments spanning reasoning, decision making, coding, and embodied tasks.", "effort": "Very High", "time_hours": 48, "usd_per_task": 8, "task_count": 1000, "best_for": ["multi"], "readiness": ["release", "leaderboard"], "budget": ["large", "unlimited"], "pros": "Holistic view of agent ability across settings.", "cons": "Complex setup; not a single narrow score.", "link": "https://agentbench.com/"}, {"slug": "benchmax", "name": "BenchMAX", "category": "multi", "what": "Tasks designed for agent teams with role specialization.", "effort": "Medium", "time_hours": 8, "usd_per_task": 2, "task_count": 300, "best_for": ["multi"], "readiness": ["prototype", "iteration", "release"], "budget": ["small", "medium"], "pros": "Directly measures delegation and coordination.", "cons": "Newer benchmark; fewer public results.", "link": "https://github.com/lyh-18/BenchMAX"}, {"slug": "simpleqa", "name": "SimpleQA", "category": "general", "what": "Fact-checking benchmark with 4,326 short, diverse questions.", "effort": "Low", "time_hours": 1, "usd_per_task": 0.03, "task_count": 4326, "best_for": ["general", "domain"], "readiness": ["prototype", "iteration"], "budget": ["tiny"], "pros": "Tests factual correctness and calibration; cheap.", "cons": "Not agentic; limited to question answering.", "link": "https://openai.com/index/simpleqa/"}, {"slug": "hellaswag", "name": "HellaSwag", "category": "general", "what": "Commonsense reasoning benchmark for everyday situations.", "effort": "Low", "time_hours": 1, "usd_per_task": 0.02, "task_count": 9954, "best_for": ["general"], "readiness": ["prototype", "iteration"], "budget": ["tiny"], "pros": "Fast benchmark for commonsense reasoning.", "cons": "Dated; easy for modern models; not agentic.", "link": "https://rowanzellers.com/hellaswag/"}, {"slug": "gsm8k", "name": "GSM8K / MATH", "category": "domain", "what": "Grade-school and competition-level math word problems.", "effort": "Low", "time_hours": 1, "usd_per_task": 0.15, "task_count": 7500, "best_for": ["domain"], "readiness": ["prototype", "iteration", "release"], "budget": ["tiny"], "pros": "Clear right/wrong scoring; cheap to run.", "cons": "Text-only; not agentic unless paired with tools.", "link": "https://github.com/openai/grade-school-math"}, {"slug": "cyber", "name": "CYBER", "category": "domain", "what": "Capture-the-flag style tasks for offensive security agents.", "effort": "High", "time_hours": 16, "usd_per_task": 8, "task_count": 200, "best_for": ["domain"], "readiness": ["release", "leaderboard"], "budget": ["medium", "large", "unlimited"], "pros": "Practical security scenarios; strong capability signal.", "cons": "Niche; dangerous if misused.", "link": "https://cyberbenchmark.com/"}, {"slug": "sciworld", "name": "SciWorld", "category": "domain", "what": "Text-based scientific experiments in a simulated lab environment.", "effort": "High", "time_hours": 12, "usd_per_task": 3, "task_count": 400, "best_for": ["domain"], "readiness": ["release", "leaderboard"], "budget": ["small", "medium", "large"], "pros": "Tests hypothesis-driven reasoning and experimentation.", "cons": "Niche domain; simulator complexity.", "link": "https://sciworld-benchmark.github.io/"}, {"slug": "coqa", "name": "CoQA", "category": "general", "what": "Conversational question-answering benchmark with context.", "effort": "Low", "time_hours": 1, "usd_per_task": 0.1, "task_count": 8000, "best_for": ["general", "voice"], "readiness": ["prototype", "iteration"], "budget": ["tiny", "small"], "pros": "Tests conversation memory and context handling.", "cons": "Not real-time voice; QA only.", "link": "https://stanfordnlp.github.io/coqa/"}, {"slug": "dstc11", "name": "DSTC11 / MultiWOZ", "category": "voice", "what": "Task-oriented dialogue benchmark for booking, scheduling, and info lookup.", "effort": "Medium", "time_hours": 4, "usd_per_task": 0.2, "task_count": 10000, "best_for": ["voice"], "readiness": ["prototype", "iteration", "release"], "budget": ["tiny", "small"], "pros": "Standard for dialogue state tracking and task completion.", "cons": "Text-based; ignores voice latency and ASR errors.", "link": "https://github.com/budzianowski/multiwoz"}, {"slug": "airbench", "name": "Air-Bench", "category": "voice", "what": "Speech-language model benchmark with audio inputs.", "effort": "Medium", "time_hours": 3, "usd_per_task": 0.3, "task_count": 3500, "best_for": ["voice"], "readiness": ["prototype", "iteration", "release"], "budget": ["tiny", "small"], "pros": "Tests audio understanding and spoken QA.", "cons": "Does not cover live telephony or turn-taking quality.", "link": "https://air-bench.org/"}, {"slug": "toolbench", "name": "ToolBench", "category": "general", "what": "API-based tool-use benchmark with 16k+ tool use instances.", "effort": "Medium", "time_hours": 6, "usd_per_task": 0.4, "task_count": 16441, "best_for": ["general", "web", "multi"], "readiness": ["prototype", "iteration", "release"], "budget": ["small", "medium"], "pros": "Large coverage of API tool use.", "cons": "Simulated APIs; may not match real API behavior.", "link": "https://github.com/OpenBMB/ToolBench"}, {"slug": "api-bank", "name": "API-Bank", "category": "general", "what": "Benchmark for evaluating LLM agents on API calls and planning.", "effort": "Medium", "time_hours": 4, "usd_per_task": 0.3, "task_count": 1605, "best_for": ["general", "web", "multi"], "readiness": ["prototype", "iteration"], "budget": ["tiny", "small"], "pros": "Focuses on API selection and chained tool use.", "cons": "Smaller than ToolBench; API set is synthetic.", "link": "https://github.com/AlibabaResearch/DAMO-ConvAI/tree/main/api-bank"}]}

def _ok(result):
    return json.dumps({"success": True, "data": result}, indent=2)

def _err(message):
    return json.dumps({"success": False, "error": message}, indent=2)


TOOL_NAME = "agent_evaluation_planner"
TOOLSET = "agents"

SCHEMA = {
  "type": "function",
  "function": {
    "name": "agent_evaluation_planner",
    "description": "Choose the right benchmark, budget, and timeline for evaluating an AI agent.",
    "parameters": {
      "type": "object",
      "properties": {
        "agent_type": {
          "type": "string",
          "description": "Agent type slug: coding, general, web, desktop, multi, voice, domain"
        },
        "readiness_level": {
          "type": "string",
          "description": "Readiness slug: prototype, iteration, release, leaderboard"
        },
        "budget": {
          "type": "string",
          "description": "Budget slug: tiny, small, medium, large, unlimited"
        }
      },
      "required": []
    }
  }
}

def _run(args):
    agent_type = args.get("agent_type", DATA["agent_types"][0]["slug"])
    readiness = args.get("readiness_level", DATA["readiness_levels"][0]["slug"])
    budget = args.get("budget", DATA["budgets"][0]["slug"])
    at = next((x for x in DATA["agent_types"] if x["slug"] == agent_type), DATA["agent_types"][0])
    rd = next((x for x in DATA["readiness_levels"] if x["slug"] == readiness), DATA["readiness_levels"][0])
    bd = next((x for x in DATA["budgets"] if x["slug"] == budget), DATA["budgets"][0])
    benchmarks = []
    for b in DATA["benchmarks"]:
        if at["slug"] in b.get("agent_types", []) and b.get("cost_usd", 0) <= bd.get("max_cost_usd", 999999):
            benchmarks.append({
                "name": b["name"],
                "cost_usd": b.get("cost_usd"),
                "effort": b.get("effort"),
                "trust": b.get("trust_level"),
                "best_for": b.get("best_for", "")
            })
    benchmarks.sort(key=lambda x: x["cost_usd"])
    return _ok({
        "agent_type": at["name"],
        "readiness_level": rd["name"],
        "budget_tier": bd["name"],
        "recommended_benchmarks": benchmarks[:5],
        "timeline": rd.get("typical_timeline", "1-2 weeks")
    })

def HANDLER(args):
    try:
        return _run(args)
    except Exception as e:
        return _err(str(e))


def register():
    """Manual registry hook. Import and call this to register with Hermes."""
    try:
        from tools.registry import registry
        registry.register(
            name=TOOL_NAME,
            toolset=TOOLSET,
            schema=SCHEMA,
            handler=HANDLER,
        )
    except ImportError:
        print("Hermes registry not found; skipping manual registration.")

if __name__ == "__main__":
    # CLI smoke test
    print(HANDLER({}))