infer_evidence_level.py

#!/usr/bin/env python3
"""
Infer evidence_level for MisakaNet lessons based on content analysis.

Usage:
    # From lesson content
    python3 scripts/infer_evidence_level.py --content "lesson content here"

    # From lesson file
    python3 scripts/infer_evidence_level.py --file path/to/lesson.md

    # With PR context
    python3 scripts/infer_evidence_level.py --file lesson.md --pr-comment
"""

import argparse
import re
import sys
from pathlib import Path

# Evidence level definitions
EVIDENCE_LEVELS = {
    "E0": "Self-reported (default)",
    "E1": "Maintainer reviewed",
    "E2": "Locally reproduced",
    "E3": "CI/sandbox verified",
    "E4": "Reused by another contributor",
}


def infer_evidence_level(content: str) -> tuple[str, str]:
    """
    Infer evidence_level from lesson content.

    Returns:
        (level, reasoning) tuple
    """
    content_lower = content.lower()

    # E4: Evidence of reuse by others
    reuse_patterns = [
        r"reused?\s+by",
        r"confirmed\s+by",
        r"verified\s+by\s+(?:another|other|different)",
        r"multiple\s+(?:contributors?|users?|agents?)",
        r"(?:2|3|4|5+)\s+(?:contributors?|users?|agents?)\s+confirmed",
        r"usage\s+report",
        r"cross[- ]?(?:node|user|agent)\s+(?:confirmation|verification|reuse)",
    ]
    for pattern in reuse_patterns:
        if re.search(pattern, content_lower):
            return "E4", f"Contains reuse evidence: '{pattern}'"

    # E3: CI/sandbox verification
    ci_patterns = [
        r"(?:ci|pipeline|workflow)\s+(?:pass|success|verified|run)",
        r"(?:github\s+actions?|gitlab\s+ci|jenkins|circle\s*ci)",
        r"ci[- ]?(?:verified|passed|success)",
        r"automated\s+(?:test|verification|check)\s+(?:pass|success)",
        r"workflow\s+(?:run|execution)\s+(?:#?\d+|link)",
        r"(?:sandbox|test[- ]?environment)\s+(?:verified|pass)",
        r"reproduc(?:ed?|ibility)\s+(?:in|via|through)\s+ci",
        r"passed?\s+(?:all\s+)?(?:ci|checks?|tests?|pipelines?)",
    ]
    for pattern in ci_patterns:
        if re.search(pattern, content_lower):
            return "E3", f"Contains CI verification evidence: '{pattern}'"

    # E2: Local reproduction evidence
    local_patterns = [
        r"locally\s+(?:reproduced?|verified?|confirmed?|tested?)",
        r"(?:i|we|author)\s+(?:reproduced?|verified?|confirmed?|tested?)\s+(?:this|it|the\s+fix)",
        r"reproduction\s+(?:steps?|log|output|result)",
        r"(?:before|after)\s+(?:fix|patch|change)\s*[:=]",
        r"(?:error|output)\s+(?:before|after)\s*(?:fix|patch|change)",
        r"smoke\s+test\s+(?:pass|success|verified)",
        r"verified?\s+(?:locally|on\s+(?:my|the)\s+(?:machine|environment|setup))",
        r"(?:my|the)\s+(?:local|dev|development)\s+(?:environment|setup)\s+(?:verified|pass)",
    ]
    for pattern in local_patterns:
        if re.search(pattern, content_lower):
            return "E2", f"Contains local reproduction evidence: '{pattern}'"

    # E1: Maintainer review evidence
    review_patterns = [
        r"maintainer\s+(?:reviewed?|approved?|accepted?)",
        r"(?:reviewed?|approved?|accepted?)\s+by\s+(?:maintainer|core\s+team)",
        r"(?:pr|pull\s+request)\s+(?:reviewed?|approved?|merged?)",
        r"(?:code\s+review|review\s+comment)",
        r"(?:lgmt|looks?\s+good\s+to\s+me|approved)",
    ]
    for pattern in review_patterns:
        if re.search(pattern, content_lower):
            return "E1", f"Contains review evidence: '{pattern}'"

    # Default to E0
    return "E0", "No evidence markers found (default level)"


def format_pr_comment(level: str, reasoning: str, content_preview: str) -> str:
    """Format evidence level as PR comment."""
    emoji = {"E0": "📝", "E1": "👀", "E2": "🔬", "E3": "✅", "E4": "🏆"}
    return f"""## Evidence Level Inference

**Suggested level: {emoji.get(level, "📝")} {level}** — {EVIDENCE_LEVELS[level]}

**Reasoning:** {reasoning}

### Content Preview
```
{content_preview[:500]}{'...' if len(content_preview) > 500 else ''}
```

### Evidence Levels Reference
| Level | Meaning | Weight |
|-------|---------|--------|
| E0 | Self-reported (default) | 0.00 |
| E1 | Maintainer reviewed | 0.25 |
| E2 | Locally reproduced | 0.50 |
| E3 | CI/sandbox verified | 0.75 |
| E4 | Reused by another contributor | 1.00 |

---
*Auto-generated by PR Genius evidence inference*"""


def main():
    parser = argparse.ArgumentParser(description="Infer evidence_level for lessons")
    parser.add_argument("--content", help="Lesson content string")
    parser.add_argument("--file", help="Path to lesson file")
    parser.add_argument("--pr-comment", action="store_true", help="Output as PR comment")
    args = parser.parse_args()

    if args.file:
        content = Path(args.file).read_text(encoding="utf-8")
    elif args.content:
        content = args.content
    else:
        print("Error: --content or --file required", file=sys.stderr)
        sys.exit(1)

    level, reasoning = infer_evidence_level(content)

    if args.pr_comment:
        print(format_pr_comment(level, reasoning, content))
    else:
        print(f"Level: {level}")
        print(f"Reasoning: {reasoning}")


if __name__ == "__main__":
    main()