Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
25 changes: 25 additions & 0 deletions Docs/infrastructure_ops.md
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,31 @@ curl http://localhost:4000/health

Adjust sizes based on expected submission volume and image sizes.

## MLflow (optional AEH observability)

Opt-in post-evaluate logging for AEH runs. Defaults: `enable-mlflow=false`,
empty `mlflow-tracking-uri`. See `config/mlflow/README.md` for security notes
(in-cluster hosts only; Route is optional/dev-only; SQLite is not HA).

```bash
# Deploy tracking server (PVC uses cluster default StorageClass)
oc apply -f config/mlflow/pvc.yaml
oc apply -f config/mlflow/service.yaml
oc apply -f config/mlflow/deployment.yaml

# Pipeline params (example)
# enable-mlflow: "true"
# mlflow-tracking-uri: http://abevalflow-mlflow.ab-eval-flow.svc.cluster.local:5000
```

Do **not** apply `config/mlflow/route.yaml` unless you also append the Route
hostname to `--allowed-hosts` in `deployment.yaml`.

The AEH runner image (`containers/agent-eval-harness/Containerfile`) bakes
`mlflow-skinny` + `pandas`. The evaluate step still pip-installs to `/tmp` only
when those imports are missing (older images). Rebuild/push the AEH image after
changing that Containerfile.

## Cleanup CronJob

Runs daily at 03:00 UTC. Configurable via environment variables:
Expand Down
67 changes: 67 additions & 0 deletions abevalflow/harbor_extensions/aeh_paths.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
"""Path helpers for AEH Harbor runs (MLflow / job-dir wiring)."""

from __future__ import annotations

import json
import logging
from pathlib import Path

logger = logging.getLogger(__name__)


def rewrite_harbor_job_dir(run_result_path: Path) -> str | None:
"""Rewrite absolute ``harbor_job_dir`` to a path relative to a shared ancestor.

AEH ``log_results._resolve_harbor_job_dir`` rejects absolute paths and
resolves relative ones against ancestors of the run directory. Harbor
``run.py`` currently stores an absolute job dir, which skips trace build.

**Side effect:** mutates ``run_result.json`` in place (overwrites
``harbor_job_dir``). Downstream consumers that need the original absolute
Harbor path should read it before calling this helper.

When the job directory is missing (moved/archived), still attempt a
relative rewrite from the absolute path string so MLflow traces can be
salvaged.

Returns the rewritten relative path, or None if unchanged / not possible.
"""
path = Path(run_result_path)
if not path.is_file():
return None

try:
data = json.loads(path.read_text()) or {}
except (json.JSONDecodeError, OSError):
return None
if not isinstance(data, dict):
return None

raw = data.get("harbor_job_dir")
if not raw:
return None

job = Path(str(raw))
if not job.is_absolute():
return None
if not job.is_dir():
logger.warning(
"harbor_job_dir does not exist (%s); attempting relative rewrite anyway",
job,
)

run_dir = path.parent.resolve()
job_resolved = job.resolve() if job.exists() else job
for root in run_dir.parents:
try:
rel = job_resolved.relative_to(root)
except ValueError:
continue
if ".." in rel.parts:
continue
rewritten = rel.as_posix()
data["harbor_job_dir"] = rewritten
path.write_text(json.dumps(data, indent=2) + "\n")
logger.info("Rewrote harbor_job_dir -> %s", rewritten)
return rewritten
return None
142 changes: 139 additions & 3 deletions abevalflow/harbor_extensions/aeh_task_enrichment.py
Original file line number Diff line number Diff line change
Expand Up @@ -10,16 +10,24 @@

This module patches generated task packages in-place before ``harbor run``.
It does not modify upstream agent-eval-harness source.

Tool interception / MCP hooks generated by AEH into ``environment/hooks/``,
``environment/tool_handlers.yaml``, and ``environment/.claude/settings.json``
are preserved. If ``generate_tasks`` skipped them, submission-level
``tool_handlers.yaml`` is copied into each task ``environment/``.
"""

from __future__ import annotations

import logging
import re
import shutil
from pathlib import Path

import yaml

logger = logging.getLogger(__name__)

_ANNOTATIONS_STAGE = """
# ABEvalFlow: stage annotations for Harbor verifier judges.
# reward.py uses case_dir basename (e.g. "workspace") under dataset.path.
Expand All @@ -30,10 +38,29 @@
fi
""".lstrip()

# Artifacts produced by agent_eval.harbor.tasks._generate_tool_interception
_INTERCEPTION_MARKERS = (
"hooks/tools.py",
"tool_handlers.yaml",
".claude/settings.json",
)

def enrich_harbor_tasks(tasks_dir: Path, *, config_path: Path) -> int:

def enrich_harbor_tasks(
tasks_dir: Path,
*,
config_path: Path,
skill_model: str | None = None,
judge_model: str | None = None,
) -> int:
"""Enrich every Harbor task package under *tasks_dir*.

Optional *skill_model* / *judge_model* rewrite ``models`` in each task's
bundled ``tests/eval.yaml`` (in-pod judges read this file). When the
submission has no ``reward:`` section, inject a weighted formula over
declared judges so all-null judge values yield 0.0 instead of AEH's
default 1.0.

Returns the number of task packages updated.
"""
config_path = Path(config_path)
Expand All @@ -42,17 +69,22 @@ def enrich_harbor_tasks(tasks_dir: Path, *, config_path: Path) -> int:
return 0

raw = yaml.safe_load(config_path.read_text()) or {}
if not isinstance(raw, dict):
raw = {}
submission_root = config_path.parent
dataset_rel = ""
dataset = raw.get("dataset") or {}
if isinstance(dataset, dict):
dataset_rel = str(dataset.get("path") or "").strip()

plugin_dirs = []
plugin_dirs: list[str] = []
runner = raw.get("runner") or {}
if isinstance(runner, dict):
plugin_dirs = list(runner.get("plugin_dirs") or [])

wants_interception = _config_wants_tool_interception(raw)
reward_inject = _reward_to_inject(raw)

updated = 0
for task_dir in sorted(p for p in tasks_dir.iterdir() if p.is_dir()):
if not (task_dir / "task.toml").is_file():
Expand All @@ -63,37 +95,97 @@ def enrich_harbor_tasks(tasks_dir: Path, *, config_path: Path) -> int:
submission_root=submission_root,
dataset_rel=dataset_rel,
plugin_dirs=plugin_dirs,
config_path=config_path,
wants_interception=wants_interception,
skill_model=skill_model,
judge_model=judge_model,
reward_inject=reward_inject,
)
updated += 1
return updated


def _reward_to_inject(raw: dict) -> dict | None:
"""Build a weighted reward block when the submission omits ``reward:``."""
if isinstance(raw.get("reward"), dict) and raw["reward"]:
return None
judges = raw.get("judges") or []
if not isinstance(judges, list):
return None
names = [str(j["name"]) for j in judges if isinstance(j, dict) and j.get("name")]
if not names:
return None
return {
"formula": "weighted",
# AEH RewardConfig defaults to [1, 5]; LLM judges in this pipeline
# emit [0, 1], so keep the range aligned or scores of 1.0 become 0.0.
"score_range": [0, 1],
"weights": {name: 1.0 for name in names},
}


def _config_wants_tool_interception(raw: dict) -> bool:
inputs = raw.get("inputs") or {}
if isinstance(inputs, dict):
tools = inputs.get("tools") or []
if isinstance(tools, list) and tools:
return True
permissions = raw.get("permissions") or {}
if isinstance(permissions, dict) and (permissions.get("deny") or permissions.get("allow")):
return True
return False


def _enrich_one_task(
task_dir: Path,
*,
case_id: str,
submission_root: Path,
dataset_rel: str,
plugin_dirs: list[str],
config_path: Path,
wants_interception: bool,
skill_model: str | None = None,
judge_model: str | None = None,
reward_inject: dict | None = None,
) -> None:
env_dir = task_dir / "environment"
env_dir.mkdir(parents=True, exist_ok=True)
tests_dir = task_dir / "tests"
tests_dir.mkdir(parents=True, exist_ok=True)

# Snapshot interception artifacts before any destructive copies
preserved = _snapshot_interception(env_dir)

# 1) annotations.yaml → environment/ (uploaded to workdir)
annotations_src = _find_annotations(submission_root, dataset_rel, case_id)
if annotations_src is not None:
shutil.copy2(annotations_src, env_dir / "annotations.yaml")

# 2) Restore dataset.path so load_case_record can resolve annotations
# 2) Patch bundled eval.yaml: dataset.path, models, reward hardening
bundled = tests_dir / "eval.yaml"
if bundled.is_file():
cfg = yaml.safe_load(bundled.read_text()) or {}
if not isinstance(cfg, dict):
cfg = {}
if isinstance(cfg.get("dataset"), dict):
cfg["dataset"]["path"] = "cases"
elif "dataset" not in cfg:
cfg["dataset"] = {"path": "cases"}
if skill_model or judge_model:
models = cfg.get("models")
if not isinstance(models, dict):
models = {}
cfg["models"] = models
if skill_model:
models["skill"] = skill_model
if judge_model:
models["judge"] = judge_model
if reward_inject is not None and not (isinstance(cfg.get("reward"), dict) and cfg["reward"]):
# Only ABEvalFlow-injected rewards get score_range [0, 1]. Do not
# rewrite submission-authored weighted rewards that omit score_range
# (AEH default [1, 5] may be intentional for Likert judges).
cfg["reward"] = reward_inject
bundled.write_text(yaml.safe_dump(cfg, sort_keys=False, allow_unicode=True))

# 3) Stage annotations in test.sh before reward runs
Expand All @@ -102,6 +194,7 @@ def _enrich_one_task(
_inject_annotations_stage(test_sh)

# 4) Copy plugin skill trees + point Harbor at them
# Only replace environment/skills/ — never wipe hooks/ or .claude/
copied_skills = False
for rel in plugin_dirs:
src = (submission_root / rel).resolve()
Expand All @@ -119,6 +212,49 @@ def _enrich_one_task(
if copied_skills:
_ensure_skills_dir_in_task_toml(task_dir / "task.toml", "/workspace/skills")

# 5) Restore / ensure tool interception + MCP hook artifacts
_restore_interception(env_dir, preserved)
_ensure_submission_tool_handlers(env_dir, config_path)
if wants_interception and not _has_interception(env_dir):
logger.warning(
"Task %s: eval.yaml declares inputs.tools/permissions but environment/ "
"has no hooks/tools.py or tool_handlers.yaml — Harbor will not intercept tools",
task_dir.name,
)


def _snapshot_interception(env_dir: Path) -> dict[str, bytes]:
"""Read interception files so enrichment cannot accidentally drop them."""
snapped: dict[str, bytes] = {}
for rel in _INTERCEPTION_MARKERS:
path = env_dir / rel
if path.is_file():
snapped[rel] = path.read_bytes()
return snapped


def _restore_interception(env_dir: Path, preserved: dict[str, bytes]) -> None:
for rel, data in preserved.items():
dest = env_dir / rel
if dest.is_file():
continue
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_bytes(data)


def _ensure_submission_tool_handlers(env_dir: Path, config_path: Path) -> None:
"""Copy submission-level tool_handlers.yaml into environment/ when missing."""
dest = env_dir / "tool_handlers.yaml"
if dest.is_file():
return
src = config_path.parent / "tool_handlers.yaml"
if src.is_file():
shutil.copy2(src, dest)


def _has_interception(env_dir: Path) -> bool:
return (env_dir / "hooks" / "tools.py").is_file() or (env_dir / "tool_handlers.yaml").is_file()


def _find_annotations(submission_root: Path, dataset_rel: str, case_id: str) -> Path | None:
candidates: list[Path] = []
Expand Down
37 changes: 37 additions & 0 deletions config/mlflow/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
# ABEvalFlow MLflow (eval-only)

Lightweight tracking server for AEH post-evaluate logging. **Not** production
telemetry HA — single replica, SQLite on a PVC.

## Apply (in-cluster CI)

```bash
oc apply -f config/mlflow/pvc.yaml
oc apply -f config/mlflow/service.yaml
oc apply -f config/mlflow/deployment.yaml
```

Pipeline tracking URI (ClusterIP):

```text
http://abevalflow-mlflow.ab-eval-flow.svc.cluster.local:5000
```

Set pipeline params `enable-mlflow=true` and `mlflow-tracking-uri` to that URL.

Client packages (`mlflow-skinny`, `pandas`) are baked into
`containers/agent-eval-harness/Containerfile`. Evaluate still pip-installs to
`/tmp` only when those imports are missing (older images).

## Security defaults

- `--allowed-hosts` lists in-cluster Service DNS + localhost only (no `*`).
- CORS wildcards are **not** enabled; browser UIs need an explicit allowlist.
- `route.yaml` is **optional / dev-only**. Do not apply it unless you also
append the Route hostname to `--allowed-hosts` in `deployment.yaml`.

## Storage

`pvc.yaml` omits `storageClassName` so the cluster default applies. Uncomment
`storageClassName: gp3` (or your class) in that file if you need a specific
provisioner.
Loading
Loading