lesson_reuse_bench.py

import argparse
import json
import os
from agents import YourAgent  # Import your agent class

# Define the task pairs
TASK_PAIRS = {
    "DCO": ["Missing sign-off", "Wrong email variant"],
    "Secret Scan": ["Token in commit", "Actions secret variant"],
    "DB Lock": ["SQLite locked", "Agent state DB variant"]
}

# Function to run the benchmark
def run_benchmark(agent, with_lessons, dry_run=False):
    scores = []
    for pair, tasks in TASK_PAIRS.items():
        task_a, task_b = tasks
        if dry_run:
            print(f"Running {pair} - Task A: {task_a}, Task B: {task_b}")
            continue
        
        # Run Task A
        result_a = agent.run(task_a)
        
        # Store the lesson from Task A
        lesson = agent.get_lesson(result_a) if with_lessons else None
        
        # Run Task B
        if with_lessons:
            result_b = agent.run(task_b, lesson=lesson)
        else:
            result_b = agent.run(task_b)
        
        # Calculate the score
        score = calculate_score(result_b)
        scores.append(score)
    
    return sum(scores) / len(scores)

# Function to calculate the score
def calculate_score(result):
    # Placeholder for actual score calculation
    return 1.0 if result == "success" else 0.0

# Main function
def main():
    parser = argparse.ArgumentParser(description="Run LessonReuseBench benchmark")
    parser.add_argument("--agent", type=str, required=True, help="Name of the agent to use")
    parser.add_argument("--compare", action="store_true", help="Run both with and without lessons")
    parser.add_argument("--dry-run", action="store_true", help="Validate structure without running the benchmark")
    args = parser.parse_args()

    # Initialize the agent
    agent = YourAgent(args.agent)

    # Run the benchmark
    if args.compare:
        with_lessons_score = run_benchmark(agent, with_lessons=True, dry_run=args.dry_run)
        without_lessons_score = run_benchmark(agent, with_lessons=False, dry_run=args.dry_run)
        delta = with_lessons_score - without_lessons_score
        print(f"With lessons: {with_lessons_score:.2f}")
        print(f"Without lessons: {without_lessons_score:.2f}")
        print(f"Delta: {delta:.2f}")
    else:
        score = run_benchmark(agent, with_lessons=True, dry_run=args.dry_run)
        print(f"Score: {score:.2f}")

if __name__ == "__main__":
    main()