retrieval_noisebench_report.json

{
  "bench_time_ms": 659,
  "benchmark": {
    "summary": {
      "num_queries": 15,
      "mean_precision_at_3": 0.444,
      "mean_precision_at_10": 0.133,
      "mean_mrr": 0.856,
      "mean_forbidden_hit_rate": 0.013
    },
    "per_query": [
      {
        "id": "dco-signoff",
        "query": "DCO check failed missing Signed-off-by",
        "relevant": [
          "dco-auto-fix-workflow"
        ],
        "forbidden": [
          "database-lock-cleanup",
          "fanuc-payload-extraction"
        ],
        "top_results": [
          "dco-auto-fix-workflow",
          "ci-dco-decouple-pythonpath-fork-pr",
          "auto-merge-ci-pipeline",
          "pr-cleanup-sop",
          "pull-request-welcome-trigger-trap"
        ],
        "precision_at_3": 0.333,
        "precision_at_10": 0.1,
        "mrr": 1.0,
        "forbidden_hit_rate": 0.0
      },
      {
        "id": "database-locked",
        "query": "database is locked SQLite",
        "relevant": [
          "agent-state-database-lock-issues-cleanup-protocol",
          "hermes-state-database-lock-issues-cleanup-protocol"
        ],
        "forbidden": [
          "dco-auto-fix-workflow",
          "fanuc-alarm-code-reference"
        ],
        "top_results": [
          "hermes-state-database-lock-issues-cleanup-protocol",
          "hx-state-database-lock-issues-cleanup-protocol",
          "agent-state-database-lock-cleanup",
          "lesson-9-redis-postgresql-replacement",
          "search-first-roadmap-loop"
        ],
        "precision_at_3": 0.333,
        "precision_at_10": 0.1,
        "mrr": 1.0,
        "forbidden_hit_rate": 0.0
      },
      {
        "id": "github-token-401",
        "query": "GitHub API 401 token expired",
        "relevant": [
          "github-api-pr-issue-management"
        ],
        "forbidden": [
          "scrapling-installation-and-usage",
          "fanuc-profinet-s7-1200-external-startup"
        ],
        "top_results": [
          "github-401-credential-lookup",
          "github-api-pr-issue-management",
          "github-actions-code-injection",
          "auto-merge-ci-pipeline",
          "auth-header-bug"
        ],
        "precision_at_3": 0.333,
        "precision_at_10": 0.1,
        "mrr": 0.5,
        "forbidden_hit_rate": 0.0
      },
      {
        "id": "pip-timeout",
        "query": "pip install HTTPS timeout proxy",
        "relevant": [
          "pip-install-timeout-ssl",
          "lesson-08-pip-https-proxy-clash"
        ],
        "forbidden": [
          "dco-auto-fix-workflow",
          "fanuc-karel-ik-fk-quaternion-guide"
        ],
        "top_results": [
          "lesson-08-pip-https-proxy-clash",
          "agent-web-access-toolchain-selection",
          "pip-install-timeout-ssl",
          "scrapling-installation-and-usage",
          "auto-merge-ci-pipeline"
        ],
        "precision_at_3": 0.667,
        "precision_at_10": 0.2,
        "mrr": 1.0,
        "forbidden_hit_rate": 0.0
      },
      {
        "id": "secret-scan",
        "query": "secret scanning token in commit",
        "relevant": [
          "lesson-11-github-commit-signing",
          "codeql-alert-dismissal-false-positive"
        ],
        "forbidden": [
          "fanuc-python-interface-fanucpy",
          "scrapling-installation-and-usage"
        ],
        "top_results": [
          "codeql-alert-dismissal-false-positive",
          "lesson-11-github-commit-signing",
          "dco-auto-fix-workflow",
          "github-actions-code-injection",
          "auto-merge-ci-pipeline"
        ],
        "precision_at_3": 0.667,
        "precision_at_10": 0.2,
        "mrr": 1.0,
        "forbidden_hit_rate": 0.0
      },
      {
        "id": "profinet-setup",
        "query": "FANUC PROFINET S7-1200 configuration",
        "relevant": [
          "fanuc-profinet-io-software-config",
          "fanuc-profinet-s7-1200-external-startup"
        ],
        "forbidden": [
          "dco-auto-fix-workflow",
          "pip-install-https-timeout-from-wsl"
        ],
        "top_results": [
          "fanuc-profinet-s7-1200-external-startup",
          "fanuc-profinet-io-software-config",
          "fanuc-profinet-32bit-real-value-transfer",
          "fanuc-dcs-safety-configuration",
          "fanuc-payload-estimation"
        ],
        "precision_at_3": 0.667,
        "precision_at_10": 0.2,
        "mrr": 1.0,
        "forbidden_hit_rate": 0.0
      },
      {
        "id": "karel-quaternion",
        "query": "KAREL quaternion IK FK gimbal lock",
        "relevant": [
          "fanuc-karel-ik-fk-quaternion-guide"
        ],
        "forbidden": [
          "github-actions-script-injection",
          "hermes-state-database-lock-issues-cleanup-protocol"
        ],
        "top_results": [
          "fanuc-karel-ik-fk-quaternion-guide",
          "fanuc-karel-kl-pose-api-reference",
          "hermes-state-database-lock-issues-cleanup-protocol",
          "hx-state-database-lock-issues-cleanup-protocol",
          "cronjob-one-shot-race-condition-duplicate-execution"
        ],
        "precision_at_3": 0.333,
        "precision_at_10": 0.1,
        "mrr": 1.0,
        "forbidden_hit_rate": 0.1
      },
      {
        "id": "github-443",
        "query": "github.com port 443 connection refused",
        "relevant": [
          "github-dns-443-block-hosts-workaround"
        ],
        "forbidden": [
          "fanuc-payload-estimation",
          "fanuc-alarm-code-reference"
        ],
        "top_results": [
          "gfw-tls-sni-block-pattern",
          "lesson-08-pip-https-proxy-clash",
          "github-dns-443-block-hosts-workaround",
          "curl-request-troubleshoot",
          "git-push-without-shell-agent"
        ],
        "precision_at_3": 0.333,
        "precision_at_10": 0.1,
        "mrr": 0.333,
        "forbidden_hit_rate": 0.0
      },
      {
        "id": "agent-web-scraping",
        "query": "agent web scraping forum anti-bot",
        "relevant": [
          "scrapling-installation-and-usage",
          "multi-forum-scraping-architecture"
        ],
        "forbidden": [
          "dco-auto-fix-workflow",
          "fanuc-karel-unit-testing-kunit"
        ],
        "top_results": [
          "scrapling-installation-and-usage",
          "multi-forum-scraping-architecture",
          "agent-reach-multi-platform-scraper",
          "agent-web-access-toolchain-selection",
          "gfw-tls-sni-blocking-tool-layer-ineffective"
        ],
        "precision_at_3": 0.667,
        "precision_at_10": 0.2,
        "mrr": 1.0,
        "forbidden_hit_rate": 0.0
      },
      {
        "id": "ci-pipeline-auto-merge",
        "query": "CI auto merge quality score DCO",
        "relevant": [
          "auto-merge-ci-pipeline"
        ],
        "forbidden": [
          "scrapling-installation-and-usage",
          "fanuc-backup-restore-guide"
        ],
        "top_results": [
          "auto-merge-ci-pipeline",
          "dco-auto-fix-workflow",
          "pr-cleanup-sop",
          "lesson-quality-requirements",
          "lesson-management-standardization"
        ],
        "precision_at_3": 0.333,
        "precision_at_10": 0.1,
        "mrr": 1.0,
        "forbidden_hit_rate": 0.0
      },
      {
        "id": "fanuc-backup",
        "query": "FANUC robot backup restore mirror",
        "relevant": [
          "fanuc-backup-restore-guide"
        ],
        "forbidden": [
          "github-actions-script-injection",
          "pip-install-https-timeout-from-wsl"
        ],
        "top_results": [
          "fanuc-backup-restore-guide",
          "fanuc-backup-payload-extraction",
          "fanuc-dcs-safety-configuration",
          "fanuc-mi-standard-software-instructions",
          "fanuc-profinet-32bit-real-value-transfer"
        ],
        "precision_at_3": 0.333,
        "precision_at_10": 0.1,
        "mrr": 1.0,
        "forbidden_hit_rate": 0.0
      },
      {
        "id": "verified-semantics",
        "query": "verified lesson badge semantics directory",
        "relevant": [],
        "forbidden": [
          "fanuc-profinet-s7-1200-external-startup",
          "hermes-state-database-lock-issues-cleanup-protocol"
        ],
        "top_results": [
          "version-management-multiple-files",
          "search-first-roadmap-loop",
          "openclaw-playwright-wsl-libnss3-libnspr4-snap-chromium",
          "fanuc-backup-payload-extraction",
          "lesson-11-github-commit-signing"
        ],
        "precision_at_3": 0.0,
        "precision_at_10": 0.0,
        "mrr": 0.0,
        "forbidden_hit_rate": 0.0
      },
      {
        "id": "git-credential-helper",
        "query": "git credential helper gh auth push failed",
        "relevant": [
          "git-credentials-and-node-id-setup",
          "git-credential-helper-gh-path-mismatch"
        ],
        "forbidden": [
          "dco-auto-fix-workflow",
          "fanuc-karel-ka-boost-architecture"
        ],
        "top_results": [
          "git-credential-helper-gh-path-mismatch",
          "lesson-06-git-push-credential-helper-403",
          "git-credentials-and-node-id-setup",
          "git-credentials-automation",
          "dco-auto-fix-workflow"
        ],
        "precision_at_3": 0.667,
        "precision_at_10": 0.2,
        "mrr": 1.0,
        "forbidden_hit_rate": 0.1
      },
      {
        "id": "codeql-dismiss",
        "query": "CodeQL alert dismiss false positive",
        "relevant": [
          "codeql-alert-dismissal-false-positive"
        ],
        "forbidden": [
          "fanuc-payload-estimation",
          "fanuc-rsr-program-select-logic"
        ],
        "top_results": [
          "codeql-alert-dismissal-false-positive",
          "github-actions-code-injection",
          "pull-request-welcome-trigger-trap",
          "auto-merge-ci-pipeline",
          "dco-auto-fix-workflow"
        ],
        "precision_at_3": 0.333,
        "precision_at_10": 0.1,
        "mrr": 1.0,
        "forbidden_hit_rate": 0.0
      },
      {
        "id": "rate-limit-429",
        "query": "GitHub API rate limit 429",
        "relevant": [
          "api-rate-limit-handling",
          "api-rate-limit-handling-best-practices"
        ],
        "forbidden": [
          "fanuc-dcs-safety-configuration",
          "fanuc-karel-uv-to-xyzwpr-pipeline"
        ],
        "top_results": [
          "api-rate-limit-handling",
          "feishu-block-api-false-success",
          "api-rate-limit-handling-best-practices",
          "github-api-pr-issue-management",
          "codeql-alert-dismissal-false-positive"
        ],
        "precision_at_3": 0.667,
        "precision_at_10": 0.2,
        "mrr": 1.0,
        "forbidden_hit_rate": 0.0
      }
    ]
  },
  "strict_vs_loose": {
    "relevant_avg_top_score": 0.882,
    "irrelevant_avg_top_score": 0.832,
    "score_gap": 0.051,
    "details": [
      {
        "id": "dco-signoff",
        "top_score": 1.055,
        "top_result": "dco-auto-fix-workflow",
        "is_relevant": true
      },
      {
        "id": "database-locked",
        "top_score": 0.958,
        "top_result": "hermes-state-database-lock-issues-cleanup-protocol",
        "is_relevant": true
      },
      {
        "id": "github-token-401",
        "top_score": 0.765,
        "top_result": "github-401-credential-lookup",
        "is_relevant": false
      },
      {
        "id": "pip-timeout",
        "top_score": 0.865,
        "top_result": "lesson-08-pip-https-proxy-clash",
        "is_relevant": true
      },
      {
        "id": "secret-scan",
        "top_score": 0.865,
        "top_result": "codeql-alert-dismissal-false-positive",
        "is_relevant": true
      },
      {
        "id": "profinet-setup",
        "top_score": 0.755,
        "top_result": "fanuc-profinet-s7-1200-external-startup",
        "is_relevant": true
      },
      {
        "id": "karel-quaternion",
        "top_score": 0.755,
        "top_result": "fanuc-karel-ik-fk-quaternion-guide",
        "is_relevant": true
      },
      {
        "id": "github-443",
        "top_score": 0.865,
        "top_result": "gfw-tls-sni-block-pattern",
        "is_relevant": false
      },
      {
        "id": "agent-web-scraping",
        "top_score": 0.891,
        "top_result": "scrapling-installation-and-usage",
        "is_relevant": true
      },
      {
        "id": "ci-pipeline-auto-merge",
        "top_score": 1.055,
        "top_result": "auto-merge-ci-pipeline",
        "is_relevant": true
      },
      {
        "id": "fanuc-backup",
        "top_score": 0.965,
        "top_result": "fanuc-backup-restore-guide",
        "is_relevant": true
      },
      {
        "id": "verified-semantics",
        "top_score": 0.865,
        "top_result": "version-management-multiple-files",
        "is_relevant": false
      },
      {
        "id": "git-credential-helper",
        "top_score": 0.765,
        "top_result": "git-credential-helper-gh-path-mismatch",
        "is_relevant": true
      },
      {
        "id": "codeql-dismiss",
        "top_score": 0.905,
        "top_result": "codeql-alert-dismissal-false-positive",
        "is_relevant": true
      },
      {
        "id": "rate-limit-429",
        "top_score": 0.755,
        "top_result": "api-rate-limit-handling",
        "is_relevant": true
      }
    ]
  }
}