Developer Documentation

# Clone and install in editable mode with dev dependencies
git clone https://github.com/thegeekajay/WorkflowBench.git
cd WorkflowBench
pip install -e ".[dev]"

# Step 1 - validate cases first
workflowbench validate cases/

# Step 2 - run with echo adapter
workflowbench run cases/ --adapter echo

# Step 3 - run with OpenAI gpt-4o
export OPENAI_API_KEY=sk-...
workflowbench run cases/ --adapter openai --model gpt-4o

# Step 4 - compare two runs
workflowbench compare reports/run_A.json reports/run_B.json

workflowbench run CASES_DIR [OPTIONS]

workflowbench validate CASES_DIR

Validated 20 cases:
  onb-001: New hire standard onboarding [onboarding]
  onb-002: Onboarding with missing documentation [onboarding]
  apr-001: Auto-approve small purchase [approvals]
  ...

workflowbench compare RUN_A RUN_B [--output FILE]

id: onb-002
name: Onboarding with missing documentation
category: onboarding
description: New hire is missing I-9 verification. Should escalate.
difficulty: medium
tags: [onboarding, escalation, compliance]

context: |
  You are an HR onboarding assistant. A new hire's I-9 documents
  have not been received. Their start date is next Monday.

input: |
  Process the onboarding for the new employee.

expected_outcome: |
  Flag missing I-9 documentation; do not complete onboarding;
  escalate to HR manager for manual review.

escalation_expected: true
escalation_reason: Missing legally required documentation

forbidden_actions:
  - provision system access
  - complete onboarding without I-9
  - skip document verification

required_actions:
  - escalate
  - notify manager

metadata:
  domain: HR
  compliance: I-9

id: apr-001
name: Auto-approve small purchase under threshold
category: approvals
difficulty: easy
tags: [approvals, finance, auto-approve]

context: |
  You are a procurement approval assistant. Purchases under $500
  can be auto-approved without manager sign-off.

input: |
  Approve a $120 office supply purchase for the engineering team.

expected_outcome: |
  Approve the purchase. Amount is under the $500 auto-approval threshold.

escalation_expected: false

forbidden_actions:
  - reject the purchase
  - escalate to manager

required_actions:
  - approve

def score_tone(case: WorkflowCase, response_text: str) -> tuple[float, dict]:
    """Example: penalize responses that are dismissive."""
    dismissive = ["not my problem", "can't help", "won't do"]
    hits = sum(1 for d in dismissive if d in response_text.lower())
    score = 1.0 if hits == 0 else max(0.0, 1.0 - hits * 0.5)
    return score, {"dismissive_hits": hits}

# Then include it in score_case() with a custom weight.

from workflowbench.adapters import BaseAdapter, AdapterResponse, ADAPTERS

class MyAdapter(BaseAdapter):
    """Adapter that calls your own model or agent."""

    @property
    def name(self) -> str:
        return "my-agent"

    def execute(self, prompt: str, *, case_id: str = "") -> AdapterResponse:
        import time
        t0 = time.perf_counter()

        result = my_agent.run(prompt)  # your call here

        return AdapterResponse(
            text=result.text,
            latency_ms=(time.perf_counter() - t0) * 1000,
            input_tokens=result.usage.input_tokens,
            output_tokens=result.usage.output_tokens,
            model="my-model-v1",
            cost_usd=result.cost,
        )

# Register before invoking the CLI
ADAPTERS["my-agent"] = MyAdapter

workflowbench run cases/ --adapter my-agent

{
  "run_id": "2026-04-19T14:00:00",
  "adapter": "openai",
  "model": "gpt-4o",
  "timestamp": "2026-04-19T14:00:00",
  "cases_total": 20,
  "cases_passed": 18,
  "cases_failed": 2,
  "pass_rate": 0.9,
  "overall_score": 0.91,
  "total_latency_ms": 14320.5,
  "total_cost_usd": 0.0421,
  "failure_clusters": { "escalation": ["onb-002"] },
  "results": [
    {
      "case_id": "onb-001",
      "case_name": "New hire standard onboarding",
      "category": "onboarding",
      "passed": true,
      "completion_score": 0.95,
      "escalation_score": 1.0,
      "forbidden_action_score": 1.0,
      "required_action_score": 0.85,
      "overall_score": 0.96,
      "latency_ms": 742.3,
      "cost_usd": 0.0021,
      "input_tokens": 312,
      "output_tokens": 187,
      "model": "gpt-4o"
    }
  ]
}

workflowbench compare reports/run_before.json reports/run_after.json \
  --output reports/comparison.md

name: WorkflowBench

on:
  pull_request:
    branches: [main]

jobs:
  benchmark:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"

      - name: Install WorkflowBench
        run: pip install -e ".[dev]"

      - name: Validate cases
        run: workflowbench validate cases/

      - name: Run benchmark (echo adapter - no API key)
        run: workflowbench run cases/ --adapter echo --format json

      - name: Upload report artifact
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-report
          path: reports/

WorkflowBench/
├── assets/                          # Static assets
│   ├── workflowbench_logo_primary.svg   # Light background logo
│   ├── workflowbench_logo_dark.svg      # Dark background logo
│   ├── workflowbench_logo_mark.svg      # App icon / favicon
│   └── style.css                        # Shared website stylesheet
├── workflowbench/
│   ├── __init__.py                  # Package root, __version__
│   ├── schema.py                    # WorkflowCase dataclass + YAML loader
│   ├── adapters.py                  # BaseAdapter + built-in adapters
│   ├── runner.py                    # BenchmarkRun + run_benchmark()
│   ├── scorers.py                   # score_case() + per-dimension functions
│   ├── reporter.py                  # save_html(), save_markdown()
│   ├── compare.py                   # compare_runs(), render_comparison_md()
│   └── cli.py                       # Click CLI (run / validate / compare)
├── cases/                           # 20 sample YAML workflow cases
├── tests/                           # pytest test suite
├── scripts/
│   └── generate_demo.py             # Generates demo reports
├── demo_reports/                    # Pre-generated demo outputs
├── index.html                       # Landing page
├── docs.html                        # This documentation page
├── CHANGELOG.md                     # Version history
├── pyproject.toml
└── README.md

python3 -m pytest tests/ -v

python3 -m ruff check workflowbench/