From d3e33bbc513169a0d363c4ae737855ed87d08684 Mon Sep 17 00:00:00 2001 From: gziv Date: Tue, 21 Jul 2026 12:50:41 +0300 Subject: [PATCH 01/12] feat(aeh): add MLflow logging, tool/MCP checks, and monitoring AEH params Wire optional post-evaluate MLflow logging, preserve Harbor tool interception artifacts, validate inputs.tools/handlers, and expose AEH params on monitoring pipelines. Add a dedicated MLflow deploy for guy-ziv-evalflow and sample mlflow.experiment on aeh-hello-world. --- .../harbor_extensions/aeh_task_enrichment.py | 84 +++++++- config/mlflow/deployment.yaml | 86 ++++++++ config/mlflow/pvc.yaml | 17 ++ config/mlflow/route.yaml | 20 ++ config/mlflow/service.yaml | 17 ++ pipeline/pipelines/ci-pipeline-dev.yaml | 14 ++ pipeline/pipelines/ci-pipeline.yaml | 14 ++ .../pipelines/monitoring-pipeline-dev.yaml | 68 ++++++- pipeline/pipelines/monitoring-pipeline.yaml | 68 ++++++- pipeline/tasks/phases/evaluate.yaml | 46 +++++ scripts/log_aeh_mlflow.py | 187 ++++++++++++++++++ scripts/misc/trigger_test_runs.sh | 40 +++- scripts/validate.py | 70 +++++++ submissions/aeh-hello-world/eval.yaml | 4 + tests/test_aeh_task_enrichment.py | 64 ++++++ tests/test_log_aeh_mlflow.py | 135 +++++++++++++ tests/test_validate_aeh.py | 33 ++++ 17 files changed, 957 insertions(+), 10 deletions(-) create mode 100644 config/mlflow/deployment.yaml create mode 100644 config/mlflow/pvc.yaml create mode 100644 config/mlflow/route.yaml create mode 100644 config/mlflow/service.yaml create mode 100644 scripts/log_aeh_mlflow.py create mode 100644 tests/test_log_aeh_mlflow.py diff --git a/abevalflow/harbor_extensions/aeh_task_enrichment.py b/abevalflow/harbor_extensions/aeh_task_enrichment.py index 353a0493..c2d9e5a8 100644 --- a/abevalflow/harbor_extensions/aeh_task_enrichment.py +++ b/abevalflow/harbor_extensions/aeh_task_enrichment.py @@ -10,16 +10,24 @@ This module patches generated task packages in-place before ``harbor run``. It does not modify upstream agent-eval-harness source. + +Tool interception / MCP hooks generated by AEH into ``environment/hooks/``, +``environment/tool_handlers.yaml``, and ``environment/.claude/settings.json`` +are preserved. If ``generate_tasks`` skipped them, submission-level +``tool_handlers.yaml`` is copied into each task ``environment/``. """ from __future__ import annotations +import logging import re import shutil from pathlib import Path import yaml +logger = logging.getLogger(__name__) + _ANNOTATIONS_STAGE = """ # ABEvalFlow: stage annotations for Harbor verifier judges. # reward.py uses case_dir basename (e.g. "workspace") under dataset.path. @@ -30,6 +38,13 @@ fi """.lstrip() +# Artifacts produced by agent_eval.harbor.tasks._generate_tool_interception +_INTERCEPTION_MARKERS = ( + "hooks/tools.py", + "tool_handlers.yaml", + ".claude/settings.json", +) + def enrich_harbor_tasks(tasks_dir: Path, *, config_path: Path) -> int: """Enrich every Harbor task package under *tasks_dir*. @@ -42,17 +57,21 @@ def enrich_harbor_tasks(tasks_dir: Path, *, config_path: Path) -> int: return 0 raw = yaml.safe_load(config_path.read_text()) or {} + if not isinstance(raw, dict): + raw = {} submission_root = config_path.parent dataset_rel = "" dataset = raw.get("dataset") or {} if isinstance(dataset, dict): dataset_rel = str(dataset.get("path") or "").strip() - plugin_dirs = [] + plugin_dirs: list[str] = [] runner = raw.get("runner") or {} if isinstance(runner, dict): plugin_dirs = list(runner.get("plugin_dirs") or []) + wants_interception = _config_wants_tool_interception(raw) + updated = 0 for task_dir in sorted(p for p in tasks_dir.iterdir() if p.is_dir()): if not (task_dir / "task.toml").is_file(): @@ -63,11 +82,25 @@ def enrich_harbor_tasks(tasks_dir: Path, *, config_path: Path) -> int: submission_root=submission_root, dataset_rel=dataset_rel, plugin_dirs=plugin_dirs, + config_path=config_path, + wants_interception=wants_interception, ) updated += 1 return updated +def _config_wants_tool_interception(raw: dict) -> bool: + inputs = raw.get("inputs") or {} + if isinstance(inputs, dict): + tools = inputs.get("tools") or [] + if isinstance(tools, list) and tools: + return True + permissions = raw.get("permissions") or {} + if isinstance(permissions, dict) and (permissions.get("deny") or permissions.get("allow")): + return True + return False + + def _enrich_one_task( task_dir: Path, *, @@ -75,12 +108,17 @@ def _enrich_one_task( submission_root: Path, dataset_rel: str, plugin_dirs: list[str], + config_path: Path, + wants_interception: bool, ) -> None: env_dir = task_dir / "environment" env_dir.mkdir(parents=True, exist_ok=True) tests_dir = task_dir / "tests" tests_dir.mkdir(parents=True, exist_ok=True) + # Snapshot interception artifacts before any destructive copies + preserved = _snapshot_interception(env_dir) + # 1) annotations.yaml → environment/ (uploaded to workdir) annotations_src = _find_annotations(submission_root, dataset_rel, case_id) if annotations_src is not None: @@ -102,6 +140,7 @@ def _enrich_one_task( _inject_annotations_stage(test_sh) # 4) Copy plugin skill trees + point Harbor at them + # Only replace environment/skills/ — never wipe hooks/ or .claude/ copied_skills = False for rel in plugin_dirs: src = (submission_root / rel).resolve() @@ -119,6 +158,49 @@ def _enrich_one_task( if copied_skills: _ensure_skills_dir_in_task_toml(task_dir / "task.toml", "/workspace/skills") + # 5) Restore / ensure tool interception + MCP hook artifacts + _restore_interception(env_dir, preserved) + _ensure_submission_tool_handlers(env_dir, config_path) + if wants_interception and not _has_interception(env_dir): + logger.warning( + "Task %s: eval.yaml declares inputs.tools/permissions but environment/ " + "has no hooks/tools.py or tool_handlers.yaml — Harbor will not intercept tools", + task_dir.name, + ) + + +def _snapshot_interception(env_dir: Path) -> dict[str, bytes]: + """Read interception files so enrichment cannot accidentally drop them.""" + snapped: dict[str, bytes] = {} + for rel in _INTERCEPTION_MARKERS: + path = env_dir / rel + if path.is_file(): + snapped[rel] = path.read_bytes() + return snapped + + +def _restore_interception(env_dir: Path, preserved: dict[str, bytes]) -> None: + for rel, data in preserved.items(): + dest = env_dir / rel + if dest.is_file(): + continue + dest.parent.mkdir(parents=True, exist_ok=True) + dest.write_bytes(data) + + +def _ensure_submission_tool_handlers(env_dir: Path, config_path: Path) -> None: + """Copy submission-level tool_handlers.yaml into environment/ when missing.""" + dest = env_dir / "tool_handlers.yaml" + if dest.is_file(): + return + src = config_path.parent / "tool_handlers.yaml" + if src.is_file(): + shutil.copy2(src, dest) + + +def _has_interception(env_dir: Path) -> bool: + return (env_dir / "hooks" / "tools.py").is_file() or (env_dir / "tool_handlers.yaml").is_file() + def _find_annotations(submission_root: Path, dataset_rel: str, case_id: str) -> Path | None: candidates: list[Path] = [] diff --git a/config/mlflow/deployment.yaml b/config/mlflow/deployment.yaml new file mode 100644 index 00000000..4200f25e --- /dev/null +++ b/config/mlflow/deployment.yaml @@ -0,0 +1,86 @@ +# Lightweight dedicated MLflow tracking server for ABEvalFlow AEH runs. +# Backend: sqlite on PVC. Artifacts: same PVC under /mlflow/artifacts. +apiVersion: apps/v1 +kind: Deployment +metadata: + name: abevalflow-mlflow + namespace: guy-ziv-evalflow + labels: + app.kubernetes.io/part-of: abevalflow + app.kubernetes.io/name: mlflow +spec: + replicas: 1 + selector: + matchLabels: + app.kubernetes.io/name: mlflow + strategy: + type: Recreate + template: + metadata: + labels: + app.kubernetes.io/name: mlflow + app.kubernetes.io/part-of: abevalflow + spec: + # OpenShift restricted-v2 assigns UID/fsGroup from the namespace range; + # do not hard-code fsGroup (it is rejected outside that range). + securityContext: + runAsNonRoot: true + containers: + - name: mlflow + image: ghcr.io/mlflow/mlflow:v3.14.0 + imagePullPolicy: IfNotPresent + ports: + - containerPort: 5000 + name: http + command: + - /bin/sh + - -ec + - | + mkdir -p /mlflow/artifacts + # Single uvicorn worker: default multi-worker forks OOM under small limits. + exec mlflow server \ + --host 0.0.0.0 \ + --port 5000 \ + --allowed-hosts '*' \ + --cors-allowed-origins '*' \ + --backend-store-uri sqlite:////mlflow/mlflow.db \ + --default-artifact-root /mlflow/artifacts \ + --serve-artifacts \ + --uvicorn-opts '--workers 1 --timeout-keep-alive 120' + volumeMounts: + - name: data + mountPath: /mlflow + - name: tmp + mountPath: /tmp + resources: + # mlflow:v3.14.0 needs ~1.5–2Gi once the ASGI app is loaded + requests: + cpu: "200m" + memory: "1Gi" + limits: + cpu: "1" + memory: "3Gi" + readinessProbe: + httpGet: + path: /api/2.0/mlflow/experiments/search?max_results=1 + port: 5000 + initialDelaySeconds: 45 + periodSeconds: 10 + failureThreshold: 18 + livenessProbe: + tcpSocket: + port: 5000 + initialDelaySeconds: 90 + periodSeconds: 30 + failureThreshold: 6 + securityContext: + allowPrivilegeEscalation: false + capabilities: + drop: ["ALL"] + readOnlyRootFilesystem: true + volumes: + - name: data + persistentVolumeClaim: + claimName: abevalflow-mlflow-data + - name: tmp + emptyDir: {} diff --git a/config/mlflow/pvc.yaml b/config/mlflow/pvc.yaml new file mode 100644 index 00000000..ab019001 --- /dev/null +++ b/config/mlflow/pvc.yaml @@ -0,0 +1,17 @@ +# Dedicated MLflow artifact + sqlite store for ABEvalFlow (dev). +# First deploy target: guy-ziv-evalflow. Promote to ab-eval-flow later if needed. +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: abevalflow-mlflow-data + namespace: guy-ziv-evalflow + labels: + app.kubernetes.io/part-of: abevalflow + app.kubernetes.io/name: mlflow +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 5Gi + storageClassName: gp3 diff --git a/config/mlflow/route.yaml b/config/mlflow/route.yaml new file mode 100644 index 00000000..625c948f --- /dev/null +++ b/config/mlflow/route.yaml @@ -0,0 +1,20 @@ +# Optional external UI / API access for guy-ziv-evalflow MLflow. +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + name: abevalflow-mlflow + namespace: guy-ziv-evalflow + labels: + app.kubernetes.io/part-of: abevalflow + app.kubernetes.io/name: mlflow +spec: + to: + kind: Service + name: abevalflow-mlflow + weight: 100 + port: + targetPort: http + tls: + termination: edge + insecureEdgeTerminationPolicy: Redirect + wildcardPolicy: None diff --git a/config/mlflow/service.yaml b/config/mlflow/service.yaml new file mode 100644 index 00000000..dba1870e --- /dev/null +++ b/config/mlflow/service.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Service +metadata: + name: abevalflow-mlflow + namespace: guy-ziv-evalflow + labels: + app.kubernetes.io/part-of: abevalflow + app.kubernetes.io/name: mlflow +spec: + type: ClusterIP + selector: + app.kubernetes.io/name: mlflow + ports: + - name: http + port: 5000 + targetPort: 5000 + protocol: TCP diff --git a/pipeline/pipelines/ci-pipeline-dev.yaml b/pipeline/pipelines/ci-pipeline-dev.yaml index bbb2318a..ccc566dc 100644 --- a/pipeline/pipelines/ci-pipeline-dev.yaml +++ b/pipeline/pipelines/ci-pipeline-dev.yaml @@ -167,6 +167,16 @@ spec: type: string default: "harbor" description: AEH execution backend (harbor, vanilla) + + # AEH MLflow (optional post-evaluate logging) + - name: enable-mlflow + type: string + default: "false" + description: When true and mlflow-tracking-uri is set, log AEH results to MLflow + - name: mlflow-tracking-uri + type: string + default: "" + description: MLflow tracking server URI (e.g. http://mlflow.ab-eval-flow.svc:5000) # Security scanning - name: security-scan type: string @@ -393,6 +403,10 @@ spec: value: $(params.aeh-image) - name: aeh-runner value: $(params.aeh-runner) + - name: enable-mlflow + value: $(params.enable-mlflow) + - name: mlflow-tracking-uri + value: $(params.mlflow-tracking-uri) workspaces: - name: source workspace: shared-workspace diff --git a/pipeline/pipelines/ci-pipeline.yaml b/pipeline/pipelines/ci-pipeline.yaml index a8412500..56aee7d1 100644 --- a/pipeline/pipelines/ci-pipeline.yaml +++ b/pipeline/pipelines/ci-pipeline.yaml @@ -167,6 +167,16 @@ spec: type: string default: "harbor" description: AEH execution backend (harbor, vanilla) + + # AEH MLflow (optional post-evaluate logging) + - name: enable-mlflow + type: string + default: "false" + description: When true and mlflow-tracking-uri is set, log AEH results to MLflow + - name: mlflow-tracking-uri + type: string + default: "" + description: MLflow tracking server URI (e.g. http://mlflow.ab-eval-flow.svc:5000) # Security scanning - name: security-scan type: string @@ -393,6 +403,10 @@ spec: value: $(params.aeh-image) - name: aeh-runner value: $(params.aeh-runner) + - name: enable-mlflow + value: $(params.enable-mlflow) + - name: mlflow-tracking-uri + value: $(params.mlflow-tracking-uri) workspaces: - name: source workspace: shared-workspace diff --git a/pipeline/pipelines/monitoring-pipeline-dev.yaml b/pipeline/pipelines/monitoring-pipeline-dev.yaml index 43cbd9ee..497aa56e 100644 --- a/pipeline/pipelines/monitoring-pipeline-dev.yaml +++ b/pipeline/pipelines/monitoring-pipeline-dev.yaml @@ -34,7 +34,7 @@ spec: - name: eval-engine type: string default: "harbor" - description: Evaluation engine (harbor, ase, mcpchecker, a2a) + description: Evaluation engine (harbor, ase, mcpchecker, a2a, aeh) # Harbor params - name: harbor-fork-url type: string @@ -113,6 +113,45 @@ spec: - name: agent-chart-url type: string default: "https://github.com/RHEcosystemAppEng/google-lightspeed-agent.git" + + # AEH (Agent-Eval-Harness) params + - name: aeh-model-override + type: string + default: "" + description: Override models.skill from eval.yaml (empty = use eval.yaml value) + - name: aeh-judge-model-override + type: string + default: "" + description: Override models.judge from eval.yaml (empty = use eval.yaml value) + - name: aeh-mode + type: string + default: "single" + description: AEH evaluation mode (single, pairwise) + - name: aeh-control-config + type: string + default: "eval-control.yaml" + description: Control variant eval.yaml filename for pairwise mode (relative to submission) + - name: aeh-treatment-config + type: string + default: "eval-treatment.yaml" + description: Treatment variant eval.yaml filename for pairwise mode (relative to submission) + - name: aeh-image + type: string + default: "quay.io/ecosystem-appeng/agent-eval-harness:v1.0.3" + description: Container image for AEH trial pods (required for harbor.run) + - name: aeh-runner + type: string + default: "harbor" + description: AEH execution backend (harbor, vanilla) + # AEH MLflow (optional post-evaluate logging) + - name: enable-mlflow + type: string + default: "false" + description: When true and mlflow-tracking-uri is set, log AEH results to MLflow + - name: mlflow-tracking-uri + type: string + default: "" + description: MLflow tracking server URI (e.g. http://mlflow.ab-eval-flow.svc:5000) # Thresholds - name: uplift-threshold type: string @@ -186,6 +225,13 @@ spec: value: "api" - name: max-generation-retries value: "0" + + - name: aeh-mode + value: $(params.aeh-mode) + - name: aeh-control-config + value: $(params.aeh-control-config) + - name: aeh-treatment-config + value: $(params.aeh-treatment-config) workspaces: - name: source workspace: shared-workspace @@ -268,6 +314,26 @@ spec: value: $(params.agent-release-name) - name: agent-chart-url value: $(params.agent-chart-url) + + # AEH params + - name: aeh-model-override + value: $(params.aeh-model-override) + - name: aeh-judge-model-override + value: $(params.aeh-judge-model-override) + - name: aeh-mode + value: $(params.aeh-mode) + - name: aeh-control-config + value: $(params.aeh-control-config) + - name: aeh-treatment-config + value: $(params.aeh-treatment-config) + - name: aeh-image + value: $(params.aeh-image) + - name: aeh-runner + value: $(params.aeh-runner) + - name: enable-mlflow + value: $(params.enable-mlflow) + - name: mlflow-tracking-uri + value: $(params.mlflow-tracking-uri) workspaces: - name: source workspace: shared-workspace diff --git a/pipeline/pipelines/monitoring-pipeline.yaml b/pipeline/pipelines/monitoring-pipeline.yaml index 938b82ca..8edfb24c 100644 --- a/pipeline/pipelines/monitoring-pipeline.yaml +++ b/pipeline/pipelines/monitoring-pipeline.yaml @@ -34,7 +34,7 @@ spec: - name: eval-engine type: string default: "harbor" - description: Evaluation engine (harbor, ase, mcpchecker, a2a) + description: Evaluation engine (harbor, ase, mcpchecker, a2a, aeh) # Harbor params - name: harbor-fork-url type: string @@ -113,6 +113,45 @@ spec: - name: agent-chart-url type: string default: "https://github.com/RHEcosystemAppEng/google-lightspeed-agent.git" + + # AEH (Agent-Eval-Harness) params + - name: aeh-model-override + type: string + default: "" + description: Override models.skill from eval.yaml (empty = use eval.yaml value) + - name: aeh-judge-model-override + type: string + default: "" + description: Override models.judge from eval.yaml (empty = use eval.yaml value) + - name: aeh-mode + type: string + default: "single" + description: AEH evaluation mode (single, pairwise) + - name: aeh-control-config + type: string + default: "eval-control.yaml" + description: Control variant eval.yaml filename for pairwise mode (relative to submission) + - name: aeh-treatment-config + type: string + default: "eval-treatment.yaml" + description: Treatment variant eval.yaml filename for pairwise mode (relative to submission) + - name: aeh-image + type: string + default: "quay.io/ecosystem-appeng/agent-eval-harness:v1.0.3" + description: Container image for AEH trial pods (required for harbor.run) + - name: aeh-runner + type: string + default: "harbor" + description: AEH execution backend (harbor, vanilla) + # AEH MLflow (optional post-evaluate logging) + - name: enable-mlflow + type: string + default: "false" + description: When true and mlflow-tracking-uri is set, log AEH results to MLflow + - name: mlflow-tracking-uri + type: string + default: "" + description: MLflow tracking server URI (e.g. http://mlflow.ab-eval-flow.svc:5000) # Thresholds - name: uplift-threshold type: string @@ -186,6 +225,13 @@ spec: value: "api" - name: max-generation-retries value: "0" + + - name: aeh-mode + value: $(params.aeh-mode) + - name: aeh-control-config + value: $(params.aeh-control-config) + - name: aeh-treatment-config + value: $(params.aeh-treatment-config) workspaces: - name: source workspace: shared-workspace @@ -268,6 +314,26 @@ spec: value: $(params.agent-release-name) - name: agent-chart-url value: $(params.agent-chart-url) + + # AEH params + - name: aeh-model-override + value: $(params.aeh-model-override) + - name: aeh-judge-model-override + value: $(params.aeh-judge-model-override) + - name: aeh-mode + value: $(params.aeh-mode) + - name: aeh-control-config + value: $(params.aeh-control-config) + - name: aeh-treatment-config + value: $(params.aeh-treatment-config) + - name: aeh-image + value: $(params.aeh-image) + - name: aeh-runner + value: $(params.aeh-runner) + - name: enable-mlflow + value: $(params.enable-mlflow) + - name: mlflow-tracking-uri + value: $(params.mlflow-tracking-uri) workspaces: - name: source workspace: shared-workspace diff --git a/pipeline/tasks/phases/evaluate.yaml b/pipeline/tasks/phases/evaluate.yaml index d0d5f919..7a6eb1c4 100644 --- a/pipeline/tasks/phases/evaluate.yaml +++ b/pipeline/tasks/phases/evaluate.yaml @@ -159,6 +159,14 @@ spec: type: string default: "harbor" description: AEH execution backend (harbor, vanilla) + - name: enable-mlflow + type: string + default: "false" + description: When true and mlflow-tracking-uri is set, log AEH results to MLflow + - name: mlflow-tracking-uri + type: string + default: "" + description: MLflow tracking server URI workspaces: - name: source description: Workspace containing the cloned submissions repository @@ -1409,6 +1417,22 @@ spec: print(f"Recommendation: {rec}") PARSE_AEH + # Optional MLflow logging (non-blocking) + if [ "$(params.enable-mlflow)" = "true" ] && [ -n "$(params.mlflow-tracking-uri)" ]; then + echo "=== AEH MLflow logging ===" + pip install --quiet --no-cache-dir 'mlflow[genai]' 2>&1 | tail -1 || true + export MLFLOW_TRACKING_URI="$(params.mlflow-tracking-uri)" + python "$PIPELINE_DIR/scripts/log_aeh_mlflow.py" \ + --enabled true \ + --tracking-uri "$(params.mlflow-tracking-uri)" \ + --run-id "$RUN_ID" \ + --config "$SUBMISSION_DIR/eval.yaml" \ + --runs-dir "$(workspaces.source.path)/reports" \ + 2>&1 || echo "Warning: MLflow logging failed (non-blocking)" + else + echo "Skipping AEH MLflow (enable-mlflow=$(params.enable-mlflow), uri set=$([ -n "$(params.mlflow-tracking-uri)" ] && echo yes || echo no))" + fi + if [ "$AEH_EXIT" -ne 0 ]; then echo "ERROR: AEH single evaluation exited $AEH_EXIT (report written if outputs existed)" exit "$AEH_EXIT" @@ -1633,6 +1657,28 @@ spec: print(f" Recommendation: {rec}") PARSE_PAIRWISE + # Optional MLflow logging for control + treatment (non-blocking) + if [ "$(params.enable-mlflow)" = "true" ] && [ -n "$(params.mlflow-tracking-uri)" ]; then + echo "=== AEH MLflow logging (pairwise) ===" + pip install --quiet --no-cache-dir 'mlflow[genai]' 2>&1 | tail -1 || true + export MLFLOW_TRACKING_URI="$(params.mlflow-tracking-uri)" + for VARIANT_RUN in "control-$RUN_ID" "treatment-$RUN_ID"; do + CFG="$SUBMISSION_DIR/$TREATMENT_CONFIG" + if [[ "$VARIANT_RUN" == control-* ]]; then + CFG="$SUBMISSION_DIR/$CONTROL_CONFIG" + fi + python "$PIPELINE_DIR/scripts/log_aeh_mlflow.py" \ + --enabled true \ + --tracking-uri "$(params.mlflow-tracking-uri)" \ + --run-id "$VARIANT_RUN" \ + --config "$CFG" \ + --runs-dir "$REPORTS_DIR" \ + 2>&1 || echo "Warning: MLflow logging failed for $VARIANT_RUN (non-blocking)" + done + else + echo "Skipping AEH MLflow (enable-mlflow=$(params.enable-mlflow))" + fi + if [ "$PAIRWISE_EXIT" -ne 0 ]; then echo "ERROR: pairwise evaluation exited $PAIRWISE_EXIT (report written if outputs existed)" exit "$PAIRWISE_EXIT" diff --git a/scripts/log_aeh_mlflow.py b/scripts/log_aeh_mlflow.py new file mode 100644 index 00000000..7d445824 --- /dev/null +++ b/scripts/log_aeh_mlflow.py @@ -0,0 +1,187 @@ +"""Log AEH Harbor run results to MLflow (post-evaluate, non-blocking caller). + +Prefers upstream AEH ``skills/eval-mlflow/scripts/log_results.py`` when present +(under ``/opt/agent-eval-harness`` or ``AGENT_EVAL_HARNESS_ROOT``). Falls back +to a minimal metrics/params logger from ``run_result.json`` + ``summary.yaml``. + +Usage:: + + python scripts/log_aeh_mlflow.py \\ + --run-id > \\ + --config /path/to/eval.yaml \\ + --runs-dir /workspace/source/reports \\ + --tracking-uri http://mlflow.example:5000 + +Skip (exit 0) when ``--enabled false`` or tracking URI is empty. +""" + +from __future__ import annotations + +import argparse +import json +import logging +import os +import subprocess +import sys +from pathlib import Path + +import yaml + +logger = logging.getLogger(__name__) + +_AEH_LOG_RESULTS_CANDIDATES = (Path("/opt/agent-eval-harness/skills/eval-mlflow/scripts/log_results.py"),) + + +def _resolve_log_results_script() -> Path | None: + root = os.environ.get("AGENT_EVAL_HARNESS_ROOT", "").strip() + candidates: list[Path] = [] + if root: + candidates.append(Path(root) / "skills/eval-mlflow/scripts/log_results.py") + candidates.extend(_AEH_LOG_RESULTS_CANDIDATES) + for path in candidates: + if path.is_file(): + return path + return None + + +def _run_aeh_log_results(*, script: Path, run_id: str, config: Path, runs_dir: Path) -> int: + env = os.environ.copy() + env["AGENT_EVAL_RUNS_DIR"] = str(runs_dir) + cmd = [ + sys.executable, + str(script), + "--run-id", + run_id, + "--config", + str(config), + ] + logger.info("Running AEH MLflow logger: %s", " ".join(cmd)) + result = subprocess.run(cmd, env=env, check=False) + return result.returncode + + +def _minimal_mlflow_log(*, run_id: str, config: Path, runs_dir: Path, tracking_uri: str) -> None: + try: + import mlflow + except ImportError as exc: + raise RuntimeError("mlflow is not installed; pip install 'mlflow[genai]' or use AEH image with mlflow") from exc + + raw = yaml.safe_load(config.read_text()) or {} + skill = str(raw.get("skill") or config.parent.name) + mlflow_cfg = raw.get("mlflow") if isinstance(raw.get("mlflow"), dict) else {} + experiment = str(mlflow_cfg.get("experiment") or skill or "abevalflow-aeh") + + run_dir = runs_dir / skill / run_id + if not run_dir.is_dir(): + # Pairwise control/treatment dirs use control- / treatment- + # when --run-id already includes the prefix. + run_dir = runs_dir / skill / run_id + if not run_dir.is_dir(): + raise FileNotFoundError(f"AEH run dir not found: {run_dir}") + + run_result: dict = {} + rr_path = run_dir / "run_result.json" + if rr_path.is_file(): + run_result = json.loads(rr_path.read_text()) or {} + + summary: dict = {} + summary_path = run_dir / "summary.yaml" + if summary_path.is_file(): + loaded = yaml.safe_load(summary_path.read_text()) or {} + if isinstance(loaded, dict): + summary = loaded + + mlflow.set_tracking_uri(tracking_uri) + mlflow.set_experiment(experiment) + with mlflow.start_run(run_name=run_id): + mlflow.log_param("skill", skill) + mlflow.log_param("run_id", run_id) + mlflow.log_param("execution_mode", run_result.get("execution_mode", "harbor")) + if run_result.get("model"): + mlflow.log_param("model", run_result["model"]) + + tokens = run_result.get("token_usage") or {} + if isinstance(tokens, dict): + for key in ("input", "output", "prompt", "completion", "total"): + if key in tokens and tokens[key] is not None: + mlflow.log_metric(f"tokens_{key}", float(tokens[key])) + # Harbor-style nested usage + for key in ("input_tokens", "output_tokens", "cache_read", "cache_create"): + if key in tokens and tokens[key] is not None: + mlflow.log_metric(f"tokens_{key}", float(tokens[key])) + + if run_result.get("cost_usd") is not None: + mlflow.log_metric("cost_usd", float(run_result["cost_usd"])) + if run_result.get("mean_reward") is not None: + mlflow.log_metric("mean_reward", float(run_result["mean_reward"])) + elif summary.get("mean_reward") is not None: + mlflow.log_metric("mean_reward", float(summary["mean_reward"])) + + if summary_path.is_file(): + mlflow.log_artifact(str(summary_path)) + if rr_path.is_file(): + mlflow.log_artifact(str(rr_path)) + report_html = run_dir / "report.html" + if report_html.is_file(): + mlflow.log_artifact(str(report_html)) + + logger.info("Minimal MLflow log complete: experiment=%s run_id=%s", experiment, run_id) + + +def main(argv: list[str] | None = None) -> int: + logging.basicConfig(level=logging.INFO, format="%(message)s") + parser = argparse.ArgumentParser(description="Log AEH results to MLflow") + parser.add_argument("--run-id", required=True) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--runs-dir", type=Path, required=True) + parser.add_argument("--tracking-uri", default=os.environ.get("MLFLOW_TRACKING_URI", "")) + parser.add_argument( + "--enabled", + default="true", + help="Set to false/0/no to skip (default: true when URI set by caller)", + ) + args = parser.parse_args(argv) + + enabled = str(args.enabled).strip().lower() in {"1", "true", "yes", "on"} + tracking_uri = (args.tracking_uri or "").strip() + if not enabled: + logger.info("MLflow logging disabled (--enabled=%s)", args.enabled) + return 0 + if not tracking_uri: + logger.info("MLflow logging skipped (empty tracking URI)") + return 0 + + os.environ["MLFLOW_TRACKING_URI"] = tracking_uri + config = args.config.resolve() + runs_dir = args.runs_dir.resolve() + if not config.is_file(): + logger.error("Config not found: %s", config) + return 1 + + script = _resolve_log_results_script() + if script is not None: + rc = _run_aeh_log_results( + script=script, + run_id=args.run_id, + config=config, + runs_dir=runs_dir, + ) + if rc == 0: + return 0 + logger.warning("AEH log_results.py exited %s; trying minimal logger", rc) + + try: + _minimal_mlflow_log( + run_id=args.run_id, + config=config, + runs_dir=runs_dir, + tracking_uri=tracking_uri, + ) + except Exception: + logger.exception("MLflow logging failed") + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/misc/trigger_test_runs.sh b/scripts/misc/trigger_test_runs.sh index b715094b..3251016a 100755 --- a/scripts/misc/trigger_test_runs.sh +++ b/scripts/misc/trigger_test_runs.sh @@ -4,20 +4,23 @@ set -euo pipefail # Trigger ABEvalFlow test PipelineRuns. # # Usage: -# ./scripts/misc/trigger_test_runs.sh [branch-name] [all|aeh|legacy] +# ./scripts/misc/trigger_test_runs.sh [branch-name] [all|aeh|legacy|aeh-mon] # # branch-name Pipeline repo revision (default: current git branch) -# mode all — Harbor/A2A/ASE monitoring+CI plus AEH single+pairwise (default) -# aeh — AEH single + pairwise only (verified smoke samples) -# legacy — original 6 Harbor/A2A/ASE runs only +# mode all — Harbor/A2A/ASE monitoring+CI plus AEH single+pairwise (default) +# aeh — AEH single + pairwise CI only (verified smoke samples) +# aeh-mon — AEH single + pairwise on monitoring pipeline +# legacy — original 6 Harbor/A2A/ASE runs only # # Env overrides: # NAMESPACE OpenShift namespace (default: guy-ziv-evalflow) # PIPELINE_CI CI pipeline name (default: abevalflow-pipeline-dev) -# PIPELINE_MONITOR Monitoring pipeline name (default: abevalflow-monitoring-pipeline) +# PIPELINE_MONITOR Monitoring pipeline name (default: abevalflow-monitoring-pipeline-dev) # LLM_API_BASE LiteLLM base URL # AEH_IMAGE AEH/Harbor trial image # SKILL_REPO skill-submissions git URL +# ENABLE_MLFLOW true/false (default false) +# MLFLOW_TRACKING_URI MLflow server URI when ENABLE_MLFLOW=true # # Verified AEH samples (skill-submissions): # single: revision=eval/aeh-hello-world-single submission-dir=aeh-hello-world-single @@ -31,11 +34,14 @@ MODE="${2:-all}" NAMESPACE="${NAMESPACE:-guy-ziv-evalflow}" PIPELINE_CI="${PIPELINE_CI:-abevalflow-pipeline-dev}" -PIPELINE_MONITOR="${PIPELINE_MONITOR:-abevalflow-monitoring-pipeline}" +PIPELINE_MONITOR="${PIPELINE_MONITOR:-abevalflow-monitoring-pipeline-dev}" LLM_API_BASE="${LLM_API_BASE:-http://litellm.ab-eval-flow.svc.cluster.local:4000}" AEH_IMAGE="${AEH_IMAGE:-quay.io/ecosystem-appeng/agent-eval-harness:v1.0.3}" SKILL_REPO="${SKILL_REPO:-https://github.com/RHEcosystemAppEng/skill-submissions.git}" LLM_MODEL="${LLM_MODEL:-claude-sonnet}" +ENABLE_MLFLOW="${ENABLE_MLFLOW:-false}" +# Dedicated ABEvalFlow MLflow in guy-ziv-evalflow (see config/mlflow/). +MLFLOW_TRACKING_URI="${MLFLOW_TRACKING_URI:-http://abevalflow-mlflow.guy-ziv-evalflow.svc.cluster.local:5000}" echo "pipeline-repo-revision: $BRANCH" echo "namespace: $NAMESPACE" @@ -77,6 +83,10 @@ spec: value: "$LLM_MODEL" - name: aeh-image value: "$AEH_IMAGE" + - name: enable-mlflow + value: "$ENABLE_MLFLOW" + - name: mlflow-tracking-uri + value: "$MLFLOW_TRACKING_URI" timeouts: pipeline: "2h" tasks: "1h30m" @@ -121,6 +131,10 @@ spec: value: "$LLM_MODEL" - name: aeh-image value: "$AEH_IMAGE" + - name: enable-mlflow + value: "$ENABLE_MLFLOW" + - name: mlflow-tracking-uri + value: "$MLFLOW_TRACKING_URI" timeouts: pipeline: "2h" tasks: "1h30m" @@ -137,6 +151,15 @@ spec: EOF } +trigger_aeh_monitoring() { + echo "=== Triggering AEH single + pairwise on monitoring pipeline ===" + local saved_ci="$PIPELINE_CI" + PIPELINE_CI="$PIPELINE_MONITOR" + # Reuse CI AEH manifests but against monitoring pipeline name + trigger_aeh + PIPELINE_CI="$saved_ci" +} + trigger_legacy() { echo "=== Triggering Harbor / A2A / ASE monitoring + CI (6 runs) ===" # Short litellm hostname for ab-eval-flow namespace; override LLM_API_BASE if needed. @@ -380,6 +403,9 @@ case "$MODE" in aeh) trigger_aeh ;; + aeh-mon) + trigger_aeh_monitoring + ;; legacy) trigger_legacy ;; @@ -388,7 +414,7 @@ case "$MODE" in trigger_aeh ;; *) - echo "Unknown mode: $MODE (use all|aeh|legacy)" >&2 + echo "Unknown mode: $MODE (use all|aeh|aeh-mon|legacy)" >&2 exit 1 ;; esac diff --git a/scripts/validate.py b/scripts/validate.py index 541af3a2..3ef45174 100644 --- a/scripts/validate.py +++ b/scripts/validate.py @@ -421,6 +421,73 @@ def _check_aeh_plugin_dirs(submission_dir: Path, eval_path: Path) -> list[str]: return errors +def _check_aeh_tool_interception(submission_dir: Path, eval_path: Path) -> list[str]: + """Validate inputs.tools / permissions / tool_handlers.yaml for Harbor MCP hooks. + + When ``inputs.tools`` is declared, each entry needs ``match``, and a + submission-level ``tool_handlers.yaml`` is required so OpenShift Harbor + trials get reliable PreToolUse patterns (heuristic fallback is weak). + """ + if not eval_path.is_file(): + return [] + try: + data = yaml.safe_load(eval_path.read_text()) or {} + except yaml.YAMLError: + return [] + if not isinstance(data, dict): + return [] + + errors: list[str] = [] + filename = eval_path.name + + permissions = data.get("permissions") + if permissions is not None: + if not isinstance(permissions, dict): + errors.append(f"{filename}: 'permissions' must be a mapping with allow/deny lists") + else: + for key in ("allow", "deny"): + if key in permissions and not isinstance(permissions[key], list): + errors.append(f"{filename}: permissions.{key} must be a list") + + inputs = data.get("inputs") + if inputs is None: + return errors + if not isinstance(inputs, dict): + errors.append(f"{filename}: 'inputs' must be a mapping") + return errors + + tools = inputs.get("tools") + if tools is None: + return errors + if not isinstance(tools, list): + errors.append(f"{filename}: inputs.tools must be a list") + return errors + + for i, entry in enumerate(tools): + if not isinstance(entry, dict): + errors.append(f"{filename}: inputs.tools[{i}] must be a mapping") + continue + if not str(entry.get("match") or "").strip(): + errors.append(f"{filename}: inputs.tools[{i}] requires a non-empty 'match' field") + + handlers = submission_dir / "tool_handlers.yaml" + if tools and not handlers.is_file(): + errors.append( + f"{filename}: inputs.tools is set but tool_handlers.yaml is missing " + f"next to {filename} (required for reliable Harbor tool/MCP interception)" + ) + elif handlers.is_file(): + try: + handlers_data = yaml.safe_load(handlers.read_text()) + except yaml.YAMLError as exc: + errors.append(f"tool_handlers.yaml is not valid YAML: {exc}") + else: + if handlers_data is not None and not isinstance(handlers_data, (dict, list)): + errors.append("tool_handlers.yaml must be a YAML mapping or list") + + return errors + + def _check_aeh_cases(submission_dir: Path) -> list[str]: """Validate cases/ directory exists with at least one case. @@ -537,17 +604,20 @@ def _check_aeh_structure( else: errors.extend(_check_aeh_eval_yaml_file(control_path)) errors.extend(_check_aeh_plugin_dirs(submission_dir, control_path)) + errors.extend(_check_aeh_tool_interception(submission_dir, control_path)) if not treatment_path.is_file(): errors.append(f"AEH pairwise: missing {treatment_config}") else: errors.extend(_check_aeh_eval_yaml_file(treatment_path)) errors.extend(_check_aeh_plugin_dirs(submission_dir, treatment_path)) + errors.extend(_check_aeh_tool_interception(submission_dir, treatment_path)) # Treatment config drives score.py pairwise — enforce contract there errors.extend(_check_aeh_pairwise_contract(treatment_path)) else: errors.extend(_check_aeh_eval_yaml(submission_dir)) errors.extend(_check_aeh_plugin_dirs(submission_dir, submission_dir / "eval.yaml")) + errors.extend(_check_aeh_tool_interception(submission_dir, submission_dir / "eval.yaml")) errors.extend(_check_aeh_cases(submission_dir)) errors.extend(_check_aeh_skill_matches_metadata(submission_dir, aeh_mode, control_config, treatment_config)) diff --git a/submissions/aeh-hello-world/eval.yaml b/submissions/aeh-hello-world/eval.yaml index bf0f8d62..a291e27e 100644 --- a/submissions/aeh-hello-world/eval.yaml +++ b/submissions/aeh-hello-world/eval.yaml @@ -3,6 +3,10 @@ skill: aeh-hello-world +# Optional: MLflow experiment name (pipeline enable-mlflow=true) +mlflow: + experiment: aeh-hello-world-single + # Runner configuration runner: type: claude-code diff --git a/tests/test_aeh_task_enrichment.py b/tests/test_aeh_task_enrichment.py index 3e1dce2d..9f6c4d81 100644 --- a/tests/test_aeh_task_enrichment.py +++ b/tests/test_aeh_task_enrichment.py @@ -98,3 +98,67 @@ def test_enrich_copies_skills_and_sets_skills_dir(tmp_path: Path): assert (task / "environment" / "skills" / "demo-skill" / "SKILL.md").is_file() toml = (task / "task.toml").read_text() assert 'skills_dir = "/workspace/skills"' in toml + + +def test_enrich_preserves_tool_interception_artifacts(tmp_path: Path): + submission = tmp_path / "submission" + skill_dir = submission / "skills" / "demo-skill" + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text("---\nname: demo-skill\n---\n# demo\n") + (submission / "cases" / "case-001").mkdir(parents=True) + (submission / "eval.yaml").write_text( + yaml.safe_dump( + { + "skill": "demo-skill", + "dataset": {"path": "cases"}, + "runner": {"type": "claude-code", "plugin_dirs": ["skills"]}, + "inputs": {"tools": [{"match": "Block MCP tools", "prompt": "deny"}]}, + "permissions": {"deny": ["mcp__*"]}, + "judges": [{"name": "exit_success"}], + } + ) + ) + (submission / "tool_handlers.yaml").write_text( + yaml.safe_dump({"handlers": [{"patterns": ["mcp__*"], "action": "deny"}]}) + ) + + tasks = tmp_path / "tasks" + task = tasks / "case-001" + _write_generated_task(task) + env = task / "environment" + (env / "hooks").mkdir() + (env / "hooks" / "tools.py").write_text("# interceptor\n") + (env / ".claude").mkdir() + (env / ".claude" / "settings.json").write_text('{"hooks":{}}') + (env / "tool_handlers.yaml").write_text("handlers: []\n") + + enrich_harbor_tasks(tasks, config_path=submission / "eval.yaml") + + assert (env / "hooks" / "tools.py").read_text() == "# interceptor\n" + assert (env / "tool_handlers.yaml").is_file() + assert (env / ".claude" / "settings.json").is_file() + assert (env / "skills" / "demo-skill" / "SKILL.md").is_file() + + +def test_enrich_copies_submission_tool_handlers_when_missing(tmp_path: Path): + submission = tmp_path / "submission" + (submission / "cases" / "case-001").mkdir(parents=True) + (submission / "eval.yaml").write_text( + yaml.safe_dump( + { + "skill": "demo", + "dataset": {"path": "cases"}, + "inputs": {"tools": [{"match": "Ask user questions"}]}, + "judges": [{"name": "exit_success"}], + } + ) + ) + (submission / "tool_handlers.yaml").write_text("handlers:\n - patterns: [AskUserQuestion]\n") + + tasks = tmp_path / "tasks" + task = tasks / "case-001" + _write_generated_task(task) + + enrich_harbor_tasks(tasks, config_path=submission / "eval.yaml") + assert (task / "environment" / "tool_handlers.yaml").is_file() + assert "AskUserQuestion" in (task / "environment" / "tool_handlers.yaml").read_text() diff --git a/tests/test_log_aeh_mlflow.py b/tests/test_log_aeh_mlflow.py new file mode 100644 index 00000000..2128d2ec --- /dev/null +++ b/tests/test_log_aeh_mlflow.py @@ -0,0 +1,135 @@ +"""Tests for scripts/log_aeh_mlflow.py.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import yaml + +from scripts.log_aeh_mlflow import main + + +def _write_run(tmp_path: Path, *, skill: str = "demo-skill", run_id: str = "run-1") -> tuple[Path, Path]: + runs = tmp_path / "reports" + run_dir = runs / skill / run_id + run_dir.mkdir(parents=True) + (run_dir / "run_result.json").write_text( + json.dumps( + { + "execution_mode": "harbor", + "mean_reward": 0.8, + "cost_usd": 0.12, + "token_usage": {"input": 100, "output": 50}, + "model": "claude-sonnet", + } + ) + ) + (run_dir / "summary.yaml").write_text(yaml.safe_dump({"mean_reward": 0.8})) + config = tmp_path / "eval.yaml" + config.write_text( + yaml.safe_dump( + { + "skill": skill, + "mlflow": {"experiment": "aeh-demo"}, + "models": {"skill": "claude-sonnet"}, + "judges": [{"name": "ok", "check": "return True, 'ok'"}], + } + ) + ) + return runs, config + + +def test_skips_when_disabled(tmp_path: Path) -> None: + runs, config = _write_run(tmp_path) + rc = main( + [ + "--run-id", + "run-1", + "--config", + str(config), + "--runs-dir", + str(runs), + "--tracking-uri", + "http://mlflow.example:5000", + "--enabled", + "false", + ] + ) + assert rc == 0 + + +def test_skips_when_uri_empty(tmp_path: Path) -> None: + runs, config = _write_run(tmp_path) + rc = main( + [ + "--run-id", + "run-1", + "--config", + str(config), + "--runs-dir", + str(runs), + "--tracking-uri", + "", + "--enabled", + "true", + ] + ) + assert rc == 0 + + +def test_minimal_logger_with_mock_mlflow(tmp_path: Path, monkeypatch) -> None: + runs, config = _write_run(tmp_path) + + class _FakeRun: + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + calls: dict[str, list] = {"params": [], "metrics": [], "artifacts": []} + + class _FakeMlflow: + def set_tracking_uri(self, uri): + calls["uri"] = uri + + def set_experiment(self, name): + calls["experiment"] = name + + def start_run(self, run_name=None): + calls["run_name"] = run_name + return _FakeRun() + + def log_param(self, k, v): + calls["params"].append((k, v)) + + def log_metric(self, k, v): + calls["metrics"].append((k, v)) + + def log_artifact(self, path): + calls["artifacts"].append(path) + + import sys + + monkeypatch.setitem(sys.modules, "mlflow", _FakeMlflow()) + + rc = main( + [ + "--run-id", + "run-1", + "--config", + str(config), + "--runs-dir", + str(runs), + "--tracking-uri", + "http://mlflow.example:5000", + "--enabled", + "true", + ] + ) + assert rc == 0 + assert calls["uri"] == "http://mlflow.example:5000" + assert calls["experiment"] == "aeh-demo" + assert ("mean_reward", 0.8) in calls["metrics"] + assert ("tokens_input", 100.0) in calls["metrics"] diff --git a/tests/test_validate_aeh.py b/tests/test_validate_aeh.py index fb1c8f6d..d0ab3a63 100644 --- a/tests/test_validate_aeh.py +++ b/tests/test_validate_aeh.py @@ -290,6 +290,39 @@ def test_single_mode_ignores_pairwise_configs(self, tmp_path): assert errors == [] +class TestAEHToolInterceptionValidation: + """inputs.tools / permissions / tool_handlers.yaml checks.""" + + def test_tools_require_match_and_handlers_file(self, tmp_path): + eval_yaml = { + **VALID_EVAL_YAML, + "inputs": {"tools": [{"prompt": "deny mcp"}]}, # missing match + } + create_aeh_submission(tmp_path, eval_yaml_content=eval_yaml) + errors = validate_submission(tmp_path, eval_engine=EvalEngine.AEH) + assert any("requires a non-empty 'match'" in e for e in errors) + assert any("tool_handlers.yaml is missing" in e for e in errors) + + def test_tools_with_handlers_passes(self, tmp_path): + eval_yaml = { + **VALID_EVAL_YAML, + "inputs": {"tools": [{"match": "Block MCP", "prompt": "deny"}]}, + "permissions": {"deny": ["mcp__*"]}, + } + create_aeh_submission(tmp_path, eval_yaml_content=eval_yaml) + (tmp_path / "tool_handlers.yaml").write_text( + yaml.dump({"handlers": [{"patterns": ["mcp__*"], "action": "deny"}]}) + ) + errors = validate_submission(tmp_path, eval_engine=EvalEngine.AEH) + assert errors == [] + + def test_invalid_permissions_type(self, tmp_path): + eval_yaml = {**VALID_EVAL_YAML, "permissions": ["mcp__*"]} + create_aeh_submission(tmp_path, eval_yaml_content=eval_yaml) + errors = validate_submission(tmp_path, eval_engine=EvalEngine.AEH) + assert any("'permissions' must be a mapping" in e for e in errors) + + class TestAEHEvalEngineEnum: """Tests for AEH in EvalEngine enum.""" From 1bd8fc7050e7d12039d9003a7ebfdfb2502a8785 Mon Sep 17 00:00:00 2001 From: gziv Date: Tue, 21 Jul 2026 12:55:20 +0300 Subject: [PATCH 02/12] fix(aeh): fall back when AEH log_results finds no mlflow package Upstream log_results.py exits 0 if mlflow is missing, which skipped our minimal logger. Detect that case, require an importable mlflow package, and install via python -m pip in the evaluate step. --- pipeline/tasks/phases/evaluate.yaml | 5 +++-- scripts/log_aeh_mlflow.py | 30 ++++++++++++++++++++++++++++- tests/test_log_aeh_mlflow.py | 29 ++++++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 3 deletions(-) diff --git a/pipeline/tasks/phases/evaluate.yaml b/pipeline/tasks/phases/evaluate.yaml index 7a6eb1c4..3e9de738 100644 --- a/pipeline/tasks/phases/evaluate.yaml +++ b/pipeline/tasks/phases/evaluate.yaml @@ -1420,7 +1420,8 @@ spec: # Optional MLflow logging (non-blocking) if [ "$(params.enable-mlflow)" = "true" ] && [ -n "$(params.mlflow-tracking-uri)" ]; then echo "=== AEH MLflow logging ===" - pip install --quiet --no-cache-dir 'mlflow[genai]' 2>&1 | tail -1 || true + # Install into the same interpreter the AEH image uses for logging. + python -m pip install --no-cache-dir 'mlflow' 2>&1 | tail -5 export MLFLOW_TRACKING_URI="$(params.mlflow-tracking-uri)" python "$PIPELINE_DIR/scripts/log_aeh_mlflow.py" \ --enabled true \ @@ -1660,7 +1661,7 @@ spec: # Optional MLflow logging for control + treatment (non-blocking) if [ "$(params.enable-mlflow)" = "true" ] && [ -n "$(params.mlflow-tracking-uri)" ]; then echo "=== AEH MLflow logging (pairwise) ===" - pip install --quiet --no-cache-dir 'mlflow[genai]' 2>&1 | tail -1 || true + python -m pip install --no-cache-dir 'mlflow' 2>&1 | tail -5 export MLFLOW_TRACKING_URI="$(params.mlflow-tracking-uri)" for VARIANT_RUN in "control-$RUN_ID" "treatment-$RUN_ID"; do CFG="$SUBMISSION_DIR/$TREATMENT_CONFIG" diff --git a/scripts/log_aeh_mlflow.py b/scripts/log_aeh_mlflow.py index 7d445824..fbc170b5 100644 --- a/scripts/log_aeh_mlflow.py +++ b/scripts/log_aeh_mlflow.py @@ -44,7 +44,21 @@ def _resolve_log_results_script() -> Path | None: return None +def _mlflow_importable() -> bool: + try: + import mlflow # noqa: F401 + except ImportError: + return False + return True + + def _run_aeh_log_results(*, script: Path, run_id: str, config: Path, runs_dir: Path) -> int: + """Run upstream log_results.py. + + Returns 0 on real success. Returns non-zero when the script no-ops because + ``mlflow`` is missing (upstream exits 0 with a message — treat as failure + so we can fall back to the minimal logger). + """ env = os.environ.copy() env["AGENT_EVAL_RUNS_DIR"] = str(runs_dir) cmd = [ @@ -56,7 +70,15 @@ def _run_aeh_log_results(*, script: Path, run_id: str, config: Path, runs_dir: P str(config), ] logger.info("Running AEH MLflow logger: %s", " ".join(cmd)) - result = subprocess.run(cmd, env=env, check=False) + result = subprocess.run(cmd, env=env, check=False, capture_output=True, text=True) + if result.stdout: + print(result.stdout, end="") + if result.stderr: + print(result.stderr, end="", file=sys.stderr) + combined = f"{result.stdout}\n{result.stderr}" + if "MLflow not installed" in combined: + logger.warning("AEH log_results.py skipped (mlflow package missing)") + return 2 return result.returncode @@ -158,6 +180,12 @@ def main(argv: list[str] | None = None) -> int: logger.error("Config not found: %s", config) return 1 + if not _mlflow_importable(): + logger.error( + "Python package 'mlflow' is not installed in this environment; install with: python -m pip install mlflow" + ) + return 1 + script = _resolve_log_results_script() if script is not None: rc = _run_aeh_log_results( diff --git a/tests/test_log_aeh_mlflow.py b/tests/test_log_aeh_mlflow.py index 2128d2ec..33968f8f 100644 --- a/tests/test_log_aeh_mlflow.py +++ b/tests/test_log_aeh_mlflow.py @@ -78,6 +78,35 @@ def test_skips_when_uri_empty(tmp_path: Path) -> None: assert rc == 0 +def test_returns_error_when_mlflow_not_importable(tmp_path: Path, monkeypatch) -> None: + runs, config = _write_run(tmp_path) + import builtins + + real_import = builtins.__import__ + + def _block_mlflow(name, *args, **kwargs): + if name == "mlflow" or name.startswith("mlflow."): + raise ImportError("blocked") + return real_import(name, *args, **kwargs) + + monkeypatch.setattr(builtins, "__import__", _block_mlflow) + rc = main( + [ + "--run-id", + "run-1", + "--config", + str(config), + "--runs-dir", + str(runs), + "--tracking-uri", + "http://mlflow.example:5000", + "--enabled", + "true", + ] + ) + assert rc == 1 + + def test_minimal_logger_with_mock_mlflow(tmp_path: Path, monkeypatch) -> None: runs, config = _write_run(tmp_path) From bc098f25adba0c67ea8916ba2ded592ac786f4be Mon Sep 17 00:00:00 2001 From: gziv Date: Tue, 21 Jul 2026 13:52:36 +0300 Subject: [PATCH 03/12] fix(aeh): harden scoring, judge model, and MLflow harbor_job_dir Prevent null judges from defaulting to reward 1.0, bake LiteLLM claude-sonnet into task packages, add the hello-world skill package, and rewrite absolute harbor_job_dir for AEH MLflow traces. --- abevalflow/harbor_extensions/aeh_paths.py | 57 +++++++++ .../harbor_extensions/aeh_task_enrichment.py | 58 ++++++++- config/mlflow/deployment.yaml | 6 +- pipeline/tasks/phases/evaluate.yaml | 113 +++++++++++++----- scripts/log_aeh_mlflow.py | 30 +++-- scripts/run_aeh.py | 19 ++- submissions/aeh-hello-world/eval.yaml | 14 ++- .../skills/aeh-hello-world/SKILL.md | 10 ++ tests/test_aeh_paths.py | 39 ++++++ tests/test_aeh_task_enrichment.py | 84 +++++++++++++ 10 files changed, 384 insertions(+), 46 deletions(-) create mode 100644 abevalflow/harbor_extensions/aeh_paths.py create mode 100644 submissions/aeh-hello-world/skills/aeh-hello-world/SKILL.md create mode 100644 tests/test_aeh_paths.py diff --git a/abevalflow/harbor_extensions/aeh_paths.py b/abevalflow/harbor_extensions/aeh_paths.py new file mode 100644 index 00000000..a4287f7a --- /dev/null +++ b/abevalflow/harbor_extensions/aeh_paths.py @@ -0,0 +1,57 @@ +"""Path helpers for AEH Harbor runs (MLflow / job-dir wiring).""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path + +logger = logging.getLogger(__name__) + + +def rewrite_harbor_job_dir(run_result_path: Path) -> str | None: + """Rewrite absolute ``harbor_job_dir`` to a path relative to a shared ancestor. + + AEH ``log_results._resolve_harbor_job_dir`` rejects absolute paths and + resolves relative ones against ancestors of the run directory. Harbor + ``run.py`` currently stores an absolute job dir, which skips trace build. + + Returns the rewritten relative path, or None if unchanged / not possible. + """ + path = Path(run_result_path) + if not path.is_file(): + return None + + try: + data = json.loads(path.read_text()) or {} + except (json.JSONDecodeError, OSError): + return None + if not isinstance(data, dict): + return None + + raw = data.get("harbor_job_dir") + if not raw: + return None + + job = Path(str(raw)) + if not job.is_absolute(): + return None + if not job.is_dir(): + logger.warning("harbor_job_dir does not exist: %s", job) + return None + + run_dir = path.parent.resolve() + job_resolved = job.resolve() + for root in run_dir.parents: + try: + rel = job_resolved.relative_to(root) + except ValueError: + continue + if ".." in rel.parts: + continue + rewritten = rel.as_posix() + data["harbor_job_dir"] = rewritten + path.write_text(json.dumps(data, indent=2) + "\n") + logger.info("Rewrote harbor_job_dir -> %s", rewritten) + return rewritten + return None diff --git a/abevalflow/harbor_extensions/aeh_task_enrichment.py b/abevalflow/harbor_extensions/aeh_task_enrichment.py index c2d9e5a8..3693a3b3 100644 --- a/abevalflow/harbor_extensions/aeh_task_enrichment.py +++ b/abevalflow/harbor_extensions/aeh_task_enrichment.py @@ -46,9 +46,21 @@ ) -def enrich_harbor_tasks(tasks_dir: Path, *, config_path: Path) -> int: +def enrich_harbor_tasks( + tasks_dir: Path, + *, + config_path: Path, + skill_model: str | None = None, + judge_model: str | None = None, +) -> int: """Enrich every Harbor task package under *tasks_dir*. + Optional *skill_model* / *judge_model* rewrite ``models`` in each task's + bundled ``tests/eval.yaml`` (in-pod judges read this file). When the + submission has no ``reward:`` section, inject a weighted formula over + declared judges so all-null judge values yield 0.0 instead of AEH's + default 1.0. + Returns the number of task packages updated. """ config_path = Path(config_path) @@ -71,6 +83,7 @@ def enrich_harbor_tasks(tasks_dir: Path, *, config_path: Path) -> int: plugin_dirs = list(runner.get("plugin_dirs") or []) wants_interception = _config_wants_tool_interception(raw) + reward_inject = _reward_to_inject(raw) updated = 0 for task_dir in sorted(p for p in tasks_dir.iterdir() if p.is_dir()): @@ -84,11 +97,34 @@ def enrich_harbor_tasks(tasks_dir: Path, *, config_path: Path) -> int: plugin_dirs=plugin_dirs, config_path=config_path, wants_interception=wants_interception, + skill_model=skill_model, + judge_model=judge_model, + reward_inject=reward_inject, ) updated += 1 return updated +def _reward_to_inject(raw: dict) -> dict | None: + """Build a weighted reward block when the submission omits ``reward:``.""" + if isinstance(raw.get("reward"), dict) and raw["reward"]: + return None + judges = raw.get("judges") or [] + if not isinstance(judges, list): + return None + names = [ + str(j["name"]) + for j in judges + if isinstance(j, dict) and j.get("name") + ] + if not names: + return None + return { + "formula": "weighted", + "weights": {name: 1.0 for name in names}, + } + + def _config_wants_tool_interception(raw: dict) -> bool: inputs = raw.get("inputs") or {} if isinstance(inputs, dict): @@ -110,6 +146,9 @@ def _enrich_one_task( plugin_dirs: list[str], config_path: Path, wants_interception: bool, + skill_model: str | None = None, + judge_model: str | None = None, + reward_inject: dict | None = None, ) -> None: env_dir = task_dir / "environment" env_dir.mkdir(parents=True, exist_ok=True) @@ -124,14 +163,29 @@ def _enrich_one_task( if annotations_src is not None: shutil.copy2(annotations_src, env_dir / "annotations.yaml") - # 2) Restore dataset.path so load_case_record can resolve annotations + # 2) Patch bundled eval.yaml: dataset.path, models, reward hardening bundled = tests_dir / "eval.yaml" if bundled.is_file(): cfg = yaml.safe_load(bundled.read_text()) or {} + if not isinstance(cfg, dict): + cfg = {} if isinstance(cfg.get("dataset"), dict): cfg["dataset"]["path"] = "cases" elif "dataset" not in cfg: cfg["dataset"] = {"path": "cases"} + if skill_model or judge_model: + models = cfg.get("models") + if not isinstance(models, dict): + models = {} + cfg["models"] = models + if skill_model: + models["skill"] = skill_model + if judge_model: + models["judge"] = judge_model + if reward_inject is not None and not ( + isinstance(cfg.get("reward"), dict) and cfg["reward"] + ): + cfg["reward"] = reward_inject bundled.write_text(yaml.safe_dump(cfg, sort_keys=False, allow_unicode=True)) # 3) Stage annotations in test.sh before reward runs diff --git a/config/mlflow/deployment.yaml b/config/mlflow/deployment.yaml index 4200f25e..bdcba1ff 100644 --- a/config/mlflow/deployment.yaml +++ b/config/mlflow/deployment.yaml @@ -37,6 +37,9 @@ spec: - -ec - | mkdir -p /mlflow/artifacts + # Proxied artifacts: clients upload via the tracking HTTP API. + # A filesystem default-artifact-root makes remote clients try to + # mkdir /mlflow locally (PermissionError under OpenShift SCC). # Single uvicorn worker: default multi-worker forks OOM under small limits. exec mlflow server \ --host 0.0.0.0 \ @@ -44,7 +47,8 @@ spec: --allowed-hosts '*' \ --cors-allowed-origins '*' \ --backend-store-uri sqlite:////mlflow/mlflow.db \ - --default-artifact-root /mlflow/artifacts \ + --default-artifact-root mlflow-artifacts:/ \ + --artifacts-destination /mlflow/artifacts \ --serve-artifacts \ --uvicorn-opts '--workers 1 --timeout-keep-alive 120' volumeMounts: diff --git a/pipeline/tasks/phases/evaluate.yaml b/pipeline/tasks/phases/evaluate.yaml index 3e9de738..4ea164e6 100644 --- a/pipeline/tasks/phases/evaluate.yaml +++ b/pipeline/tasks/phases/evaluate.yaml @@ -1264,8 +1264,10 @@ spec: memory: "512Mi" limits: cpu: "1000m" - memory: "1Gi" + memory: "2Gi" env: + - name: HOME + value: /tmp - name: AGENT_EVAL_K8S_NAMESPACE valueFrom: fieldRef: @@ -1327,11 +1329,16 @@ spec: if isinstance(models, str): print(models) else: - print(models.get('skill', 'claude-sonnet-4-5')) + print(models.get('skill', 'claude-sonnet')) ") fi JUDGE_OVERRIDE="$(params.aeh-judge-model-override)" + # Default judges to the resolved skill model so LiteLLM aliases stay aligned + # (eval.yaml may still name a model the proxy does not expose). + if [ -z "$JUDGE_OVERRIDE" ] && [ -n "$MODEL" ]; then + JUDGE_OVERRIDE="$MODEL" + fi echo "Submission: $SUBMISSION_DIR" echo "Output: $OUT_DIR" @@ -1417,19 +1424,38 @@ spec: print(f"Recommendation: {rec}") PARSE_AEH - # Optional MLflow logging (non-blocking) + # Optional MLflow logging (non-blocking). + # AEH image site-packages is read-only under OpenShift SCC — install to /tmp. if [ "$(params.enable-mlflow)" = "true" ] && [ -n "$(params.mlflow-tracking-uri)" ]; then echo "=== AEH MLflow logging ===" - # Install into the same interpreter the AEH image uses for logging. - python -m pip install --no-cache-dir 'mlflow' 2>&1 | tail -5 - export MLFLOW_TRACKING_URI="$(params.mlflow-tracking-uri)" - python "$PIPELINE_DIR/scripts/log_aeh_mlflow.py" \ - --enabled true \ - --tracking-uri "$(params.mlflow-tracking-uri)" \ - --run-id "$RUN_ID" \ - --config "$SUBMISSION_DIR/eval.yaml" \ - --runs-dir "$(workspaces.source.path)/reports" \ - 2>&1 || echo "Warning: MLflow logging failed (non-blocking)" + MLFLOW_PKG_DIR="/tmp/mlflow-pkgs" + mkdir -p "$MLFLOW_PKG_DIR" + export PYTHONPATH="${MLFLOW_PKG_DIR}${PYTHONPATH:+:$PYTHONPATH}" + # pandas is required by AEH log_results.py (mlflow.log_table). + if ! python -c "import mlflow, pandas" 2>/dev/null; then + echo "Installing mlflow-skinny + pandas into ${MLFLOW_PKG_DIR} ..." + # Do not let pip failure abort the step (set -e / pipefail). + set +e + python -m pip install --no-cache-dir --target "$MLFLOW_PKG_DIR" 'mlflow-skinny' 'pandas' 2>&1 | tee /tmp/mlflow-pip.log | tail -30 + PIP_RC=${PIPESTATUS[0]} + set -e + if [ "$PIP_RC" -ne 0 ]; then + echo "WARN: mlflow-skinny/pandas install failed (rc=$PIP_RC)" + tail -40 /tmp/mlflow-pip.log || true + fi + fi + if python -c "import mlflow, pandas; print('mlflow OK', mlflow.__version__, 'pandas', pandas.__version__)"; then + export MLFLOW_TRACKING_URI="$(params.mlflow-tracking-uri)" + python "$PIPELINE_DIR/scripts/log_aeh_mlflow.py" \ + --enabled true \ + --tracking-uri "$(params.mlflow-tracking-uri)" \ + --run-id "$RUN_ID" \ + --config "$SUBMISSION_DIR/eval.yaml" \ + --runs-dir "$(workspaces.source.path)/reports" \ + 2>&1 || echo "Warning: MLflow logging failed (non-blocking)" + else + echo "Warning: mlflow/pandas not importable; skipping MLflow logging" + fi else echo "Skipping AEH MLflow (enable-mlflow=$(params.enable-mlflow), uri set=$([ -n "$(params.mlflow-tracking-uri)" ] && echo yes || echo no))" fi @@ -1455,8 +1481,10 @@ spec: memory: "512Mi" limits: cpu: "1000m" - memory: "1Gi" + memory: "2Gi" env: + - name: HOME + value: /tmp - name: AGENT_EVAL_K8S_NAMESPACE valueFrom: fieldRef: @@ -1473,7 +1501,7 @@ spec: script: | #!/usr/bin/env bash set -euo pipefail - + EVAL_ENGINE="$(params.eval-engine)" AEH_MODE="$(params.aeh-mode)" if [ "$EVAL_ENGINE" != "aeh" ] || [ "$AEH_MODE" != "pairwise" ]; then @@ -1502,11 +1530,14 @@ spec: if isinstance(models, str): print(models) else: - print(models.get('skill', 'claude-sonnet-4-5')) + print(models.get('skill', 'claude-sonnet')) ") fi JUDGE_OVERRIDE="$(params.aeh-judge-model-override)" + if [ -z "$JUDGE_OVERRIDE" ] && [ -n "$MODEL" ]; then + JUDGE_OVERRIDE="$MODEL" + fi AEH_IMAGE="$(params.aeh-image)" AEH_RUNNER="$(params.aeh-runner)" @@ -1518,6 +1549,7 @@ spec: echo "Control config: $CONTROL_CONFIG" echo "Treatment config: $TREATMENT_CONFIG" echo "Model: $MODEL" + echo "Judge override: ${JUDGE_OVERRIDE:-none}" echo "Runner: $AEH_RUNNER" # Add pipeline dir to PYTHONPATH so abevalflow module is importable @@ -1658,24 +1690,43 @@ spec: print(f" Recommendation: {rec}") PARSE_PAIRWISE - # Optional MLflow logging for control + treatment (non-blocking) + # Optional MLflow logging for control + treatment (non-blocking). + # AEH image site-packages is read-only under OpenShift SCC — install to /tmp. if [ "$(params.enable-mlflow)" = "true" ] && [ -n "$(params.mlflow-tracking-uri)" ]; then echo "=== AEH MLflow logging (pairwise) ===" - python -m pip install --no-cache-dir 'mlflow' 2>&1 | tail -5 - export MLFLOW_TRACKING_URI="$(params.mlflow-tracking-uri)" - for VARIANT_RUN in "control-$RUN_ID" "treatment-$RUN_ID"; do - CFG="$SUBMISSION_DIR/$TREATMENT_CONFIG" - if [[ "$VARIANT_RUN" == control-* ]]; then - CFG="$SUBMISSION_DIR/$CONTROL_CONFIG" + MLFLOW_PKG_DIR="/tmp/mlflow-pkgs" + mkdir -p "$MLFLOW_PKG_DIR" + export PYTHONPATH="${MLFLOW_PKG_DIR}${PYTHONPATH:+:$PYTHONPATH}" + # pandas is required by AEH log_results.py (mlflow.log_table). + if ! python -c "import mlflow, pandas" 2>/dev/null; then + echo "Installing mlflow-skinny + pandas into ${MLFLOW_PKG_DIR} ..." + set +e + python -m pip install --no-cache-dir --target "$MLFLOW_PKG_DIR" 'mlflow-skinny' 'pandas' 2>&1 | tee /tmp/mlflow-pip.log | tail -30 + PIP_RC=${PIPESTATUS[0]} + set -e + if [ "$PIP_RC" -ne 0 ]; then + echo "WARN: mlflow-skinny/pandas install failed (rc=$PIP_RC)" + tail -40 /tmp/mlflow-pip.log || true fi - python "$PIPELINE_DIR/scripts/log_aeh_mlflow.py" \ - --enabled true \ - --tracking-uri "$(params.mlflow-tracking-uri)" \ - --run-id "$VARIANT_RUN" \ - --config "$CFG" \ - --runs-dir "$REPORTS_DIR" \ - 2>&1 || echo "Warning: MLflow logging failed for $VARIANT_RUN (non-blocking)" - done + fi + if python -c "import mlflow, pandas; print('mlflow OK', mlflow.__version__, 'pandas', pandas.__version__)"; then + export MLFLOW_TRACKING_URI="$(params.mlflow-tracking-uri)" + for VARIANT_RUN in "control-$RUN_ID" "treatment-$RUN_ID"; do + CFG="$SUBMISSION_DIR/$TREATMENT_CONFIG" + if [[ "$VARIANT_RUN" == control-* ]]; then + CFG="$SUBMISSION_DIR/$CONTROL_CONFIG" + fi + python "$PIPELINE_DIR/scripts/log_aeh_mlflow.py" \ + --enabled true \ + --tracking-uri "$(params.mlflow-tracking-uri)" \ + --run-id "$VARIANT_RUN" \ + --config "$CFG" \ + --runs-dir "$REPORTS_DIR" \ + 2>&1 || echo "Warning: MLflow logging failed for $VARIANT_RUN (non-blocking)" + done + else + echo "Warning: mlflow/pandas not importable; skipping MLflow logging" + fi else echo "Skipping AEH MLflow (enable-mlflow=$(params.enable-mlflow))" fi diff --git a/scripts/log_aeh_mlflow.py b/scripts/log_aeh_mlflow.py index fbc170b5..80063782 100644 --- a/scripts/log_aeh_mlflow.py +++ b/scripts/log_aeh_mlflow.py @@ -139,13 +139,15 @@ def _minimal_mlflow_log(*, run_id: str, config: Path, runs_dir: Path, tracking_u elif summary.get("mean_reward") is not None: mlflow.log_metric("mean_reward", float(summary["mean_reward"])) - if summary_path.is_file(): - mlflow.log_artifact(str(summary_path)) - if rr_path.is_file(): - mlflow.log_artifact(str(rr_path)) - report_html = run_dir / "report.html" - if report_html.is_file(): - mlflow.log_artifact(str(report_html)) + for artifact in (summary_path, rr_path, run_dir / "report.html"): + if not artifact.is_file(): + continue + try: + mlflow.log_artifact(str(artifact)) + except Exception: + # Metrics/params already recorded; artifacts need a proxied + # artifact root (mlflow-artifacts:/) on the tracking server. + logger.warning("MLflow artifact upload failed for %s", artifact, exc_info=True) logger.info("Minimal MLflow log complete: experiment=%s run_id=%s", experiment, run_id) @@ -186,6 +188,20 @@ def main(argv: list[str] | None = None) -> int: ) return 1 + # Belt-and-suspenders: AEH log_results rejects absolute harbor_job_dir. + try: + from abevalflow.harbor_extensions.aeh_paths import rewrite_harbor_job_dir + + raw_cfg = yaml.safe_load(config.read_text()) or {} + skill = str(raw_cfg.get("skill") or config.parent.name) + rr = runs_dir / skill / args.run_id / "run_result.json" + if rr.is_file(): + rewritten = rewrite_harbor_job_dir(rr) + if rewritten: + logger.info("Rewrote harbor_job_dir -> %s", rewritten) + except Exception: + logger.warning("harbor_job_dir rewrite failed; continuing", exc_info=True) + script = _resolve_log_results_script() if script is not None: rc = _run_aeh_log_results( diff --git a/scripts/run_aeh.py b/scripts/run_aeh.py index 4fb5460f..db783c8e 100644 --- a/scripts/run_aeh.py +++ b/scripts/run_aeh.py @@ -514,6 +514,16 @@ def _execute( except Exception as exc: # noqa: BLE001 — best-effort bridge; don't hide harbor rc print(f"WARN: failed to materialize AEH case outputs: {exc}") + # AEH MLflow logger rejects absolute harbor_job_dir; store workspace-relative. + try: + from abevalflow.harbor_extensions.aeh_paths import rewrite_harbor_job_dir + + rewritten = rewrite_harbor_job_dir(Path(output) / "run_result.json") + if rewritten: + print(f"Rewrote harbor_job_dir -> {rewritten}") + except Exception as exc: # noqa: BLE001 — best-effort; don't hide harbor rc + print(f"WARN: failed to rewrite harbor_job_dir: {exc}") + return result.returncode def _prepare_enriched_tasks(self, config: Path, tasks_dir: Path) -> None: @@ -545,8 +555,13 @@ def _prepare_enriched_tasks(self, config: Path, tasks_dir: Path) -> None: self.image, judge_model=self.judge_model, ) - n = enrich_harbor_tasks(tasks_dir, config_path=Path(config)) - print(f"Enriched {n} Harbor task package(s) (skills + annotations)") + n = enrich_harbor_tasks( + tasks_dir, + config_path=Path(config), + skill_model=self.model, + judge_model=self.judge_model, + ) + print(f"Enriched {n} Harbor task package(s) (skills + annotations + models/reward)") def _patch_eval_config_for_openshift(self, config: Path, tasks_dir: Path | None) -> Path: """Patch eval.yaml for OpenShift emptyDir support. diff --git a/submissions/aeh-hello-world/eval.yaml b/submissions/aeh-hello-world/eval.yaml index a291e27e..1b6c7911 100644 --- a/submissions/aeh-hello-world/eval.yaml +++ b/submissions/aeh-hello-world/eval.yaml @@ -11,18 +11,26 @@ mlflow: runner: type: claude-code effort: low + plugin_dirs: [skills] settings: permission_mode: bypassPermissions -# Model configuration +# Model configuration (LiteLLM in-cluster alias; override via pipeline params if needed) models: - skill: claude-sonnet-4-5 - judge: claude-sonnet-4-5 + skill: claude-sonnet + judge: claude-sonnet # Dataset location (relative to submission) dataset: path: cases +# When all judges fail/null, weighted → 0.0 (AEH default average would yield 1.0) +reward: + formula: weighted + weights: + correctness: 1.0 + helpfulness: 1.0 + # Judges evaluate each case judges: - name: correctness diff --git a/submissions/aeh-hello-world/skills/aeh-hello-world/SKILL.md b/submissions/aeh-hello-world/skills/aeh-hello-world/SKILL.md new file mode 100644 index 00000000..dcbe215c --- /dev/null +++ b/submissions/aeh-hello-world/skills/aeh-hello-world/SKILL.md @@ -0,0 +1,10 @@ +--- +name: aeh-hello-world +description: Minimal sample skill that writes a Hello World Python program +--- + +# aeh-hello-world + +When invoked, create a simple Python program that prints `Hello, World!` to the console. + +Write the program to a `.py` file in the workspace (for example `hello.py`) and keep it short and idiomatic. diff --git a/tests/test_aeh_paths.py b/tests/test_aeh_paths.py new file mode 100644 index 00000000..9d62268c --- /dev/null +++ b/tests/test_aeh_paths.py @@ -0,0 +1,39 @@ +"""Tests for AEH harbor_job_dir path rewriting.""" + +from __future__ import annotations + +import json +from pathlib import Path + +from abevalflow.harbor_extensions.aeh_paths import rewrite_harbor_job_dir + + +def test_rewrite_absolute_harbor_job_dir_to_relative(tmp_path: Path): + source = tmp_path / "source" + job = source / "_eval_tmp" / "aeh-jobs" / "2026-07-21__10-18-16" + job.mkdir(parents=True) + run_dir = source / "reports" / "aeh-hello-world" / "run-1" + run_dir.mkdir(parents=True) + rr = run_dir / "run_result.json" + rr.write_text(json.dumps({"harbor_job_dir": str(job.resolve()), "mean_reward": 0.0})) + + rewritten = rewrite_harbor_job_dir(rr) + assert rewritten == "_eval_tmp/aeh-jobs/2026-07-21__10-18-16" + + data = json.loads(rr.read_text()) + assert data["harbor_job_dir"] == rewritten + assert not Path(data["harbor_job_dir"]).is_absolute() + + +def test_rewrite_leaves_relative_path_alone(tmp_path: Path): + run_dir = tmp_path / "reports" / "skill" / "run-1" + run_dir.mkdir(parents=True) + rr = run_dir / "run_result.json" + rr.write_text(json.dumps({"harbor_job_dir": "_eval_tmp/aeh-jobs/job-1"})) + + assert rewrite_harbor_job_dir(rr) is None + assert json.loads(rr.read_text())["harbor_job_dir"] == "_eval_tmp/aeh-jobs/job-1" + + +def test_rewrite_missing_file_returns_none(tmp_path: Path): + assert rewrite_harbor_job_dir(tmp_path / "missing.json") is None diff --git a/tests/test_aeh_task_enrichment.py b/tests/test_aeh_task_enrichment.py index 9f6c4d81..c52ea23c 100644 --- a/tests/test_aeh_task_enrichment.py +++ b/tests/test_aeh_task_enrichment.py @@ -162,3 +162,87 @@ def test_enrich_copies_submission_tool_handlers_when_missing(tmp_path: Path): enrich_harbor_tasks(tasks, config_path=submission / "eval.yaml") assert (task / "environment" / "tool_handlers.yaml").is_file() assert "AskUserQuestion" in (task / "environment" / "tool_handlers.yaml").read_text() + + +def test_enrich_patches_models_and_injects_weighted_reward(tmp_path: Path): + submission = tmp_path / "submission" + (submission / "cases" / "case-001").mkdir(parents=True) + (submission / "eval.yaml").write_text( + yaml.safe_dump( + { + "skill": "demo", + "dataset": {"path": "cases"}, + "models": {"skill": "old-skill", "judge": "old-judge"}, + "judges": [ + {"name": "correctness", "type": "llm"}, + {"name": "helpfulness", "type": "llm"}, + ], + } + ) + ) + + tasks = tmp_path / "tasks" + task = tasks / "case-001" + _write_generated_task(task) + (task / "tests" / "eval.yaml").write_text( + yaml.safe_dump( + { + "dataset": {"path": ""}, + "models": {"skill": "old-skill", "judge": "old-judge"}, + "judges": [ + {"name": "correctness", "type": "llm"}, + {"name": "helpfulness", "type": "llm"}, + ], + }, + sort_keys=False, + ) + ) + + enrich_harbor_tasks( + tasks, + config_path=submission / "eval.yaml", + skill_model="claude-sonnet", + judge_model="claude-sonnet", + ) + + bundled = yaml.safe_load((task / "tests" / "eval.yaml").read_text()) + assert bundled["models"]["skill"] == "claude-sonnet" + assert bundled["models"]["judge"] == "claude-sonnet" + assert bundled["reward"]["formula"] == "weighted" + assert bundled["reward"]["weights"] == { + "correctness": 1.0, + "helpfulness": 1.0, + } + + +def test_enrich_preserves_explicit_reward_section(tmp_path: Path): + submission = tmp_path / "submission" + (submission / "cases" / "case-001").mkdir(parents=True) + (submission / "eval.yaml").write_text( + yaml.safe_dump( + { + "skill": "demo", + "dataset": {"path": "cases"}, + "reward": {"formula": "weighted", "weights": {"correctness": 2.0}}, + "judges": [{"name": "correctness"}, {"name": "helpfulness"}], + } + ) + ) + + tasks = tmp_path / "tasks" + task = tasks / "case-001" + _write_generated_task(task) + (task / "tests" / "eval.yaml").write_text( + yaml.safe_dump( + { + "dataset": {"path": ""}, + "reward": {"formula": "weighted", "weights": {"correctness": 2.0}}, + "judges": [{"name": "correctness"}, {"name": "helpfulness"}], + }, + sort_keys=False, + ) + ) + + enrich_harbor_tasks(tasks, config_path=submission / "eval.yaml") + bundled = yaml.safe_load((task / "tests" / "eval.yaml").read_text()) + assert bundled["reward"]["weights"] == {"correctness": 2.0} From 0e8ad20b2880dd462be5b80e0597a76a060e2d2b Mon Sep 17 00:00:00 2001 From: gziv Date: Tue, 21 Jul 2026 14:08:03 +0300 Subject: [PATCH 04/12] fix(aeh): set reward score_range to [0, 1] for LLM judges AEH defaults weighted rewards to Likert [1, 5], which mapped [0, 1] judge scores of 1.0 to overall reward 0.0. --- .../harbor_extensions/aeh_task_enrichment.py | 9 +++++ submissions/aeh-hello-world/eval.yaml | 5 ++- tests/test_aeh_task_enrichment.py | 40 +++++++++++++++++++ 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/abevalflow/harbor_extensions/aeh_task_enrichment.py b/abevalflow/harbor_extensions/aeh_task_enrichment.py index 3693a3b3..d8f7003a 100644 --- a/abevalflow/harbor_extensions/aeh_task_enrichment.py +++ b/abevalflow/harbor_extensions/aeh_task_enrichment.py @@ -121,6 +121,9 @@ def _reward_to_inject(raw: dict) -> dict | None: return None return { "formula": "weighted", + # AEH RewardConfig defaults to [1, 5]; LLM judges in this pipeline + # emit [0, 1], so keep the range aligned or scores of 1.0 become 0.0. + "score_range": [0, 1], "weights": {name: 1.0 for name in names}, } @@ -186,6 +189,12 @@ def _enrich_one_task( isinstance(cfg.get("reward"), dict) and cfg["reward"] ): cfg["reward"] = reward_inject + # Harden existing reward blocks that omit score_range (AEH default [1,5] + # collapses [0,1] judge scores of 1.0 to reward 0.0). + reward = cfg.get("reward") + if isinstance(reward, dict) and reward.get("formula") == "weighted": + if "score_range" not in reward and "raw" not in reward: + reward["score_range"] = [0, 1] bundled.write_text(yaml.safe_dump(cfg, sort_keys=False, allow_unicode=True)) # 3) Stage annotations in test.sh before reward runs diff --git a/submissions/aeh-hello-world/eval.yaml b/submissions/aeh-hello-world/eval.yaml index 1b6c7911..b48f5a09 100644 --- a/submissions/aeh-hello-world/eval.yaml +++ b/submissions/aeh-hello-world/eval.yaml @@ -24,9 +24,12 @@ models: dataset: path: cases -# When all judges fail/null, weighted → 0.0 (AEH default average would yield 1.0) +# When all judges fail/null, weighted → 0.0 (AEH default average would yield 1.0). +# score_range must match judge scale: AEH defaults to [1, 5], which maps a +# [0, 1] score of 1.0 to reward 0.0. reward: formula: weighted + score_range: [0, 1] weights: correctness: 1.0 helpfulness: 1.0 diff --git a/tests/test_aeh_task_enrichment.py b/tests/test_aeh_task_enrichment.py index c52ea23c..e9a4e7ee 100644 --- a/tests/test_aeh_task_enrichment.py +++ b/tests/test_aeh_task_enrichment.py @@ -209,12 +209,52 @@ def test_enrich_patches_models_and_injects_weighted_reward(tmp_path: Path): assert bundled["models"]["skill"] == "claude-sonnet" assert bundled["models"]["judge"] == "claude-sonnet" assert bundled["reward"]["formula"] == "weighted" + assert bundled["reward"]["score_range"] == [0, 1] assert bundled["reward"]["weights"] == { "correctness": 1.0, "helpfulness": 1.0, } +def test_enrich_adds_score_range_to_weighted_reward_without_one(tmp_path: Path): + submission = tmp_path / "submission" + (submission / "cases" / "case-001").mkdir(parents=True) + (submission / "eval.yaml").write_text( + yaml.safe_dump( + { + "skill": "demo", + "dataset": {"path": "cases"}, + "reward": { + "formula": "weighted", + "weights": {"correctness": 1.0, "helpfulness": 1.0}, + }, + "judges": [{"name": "correctness"}, {"name": "helpfulness"}], + } + ) + ) + + tasks = tmp_path / "tasks" + task = tasks / "case-001" + _write_generated_task(task) + (task / "tests" / "eval.yaml").write_text( + yaml.safe_dump( + { + "dataset": {"path": ""}, + "reward": { + "formula": "weighted", + "weights": {"correctness": 1.0, "helpfulness": 1.0}, + }, + "judges": [{"name": "correctness"}, {"name": "helpfulness"}], + }, + sort_keys=False, + ) + ) + + enrich_harbor_tasks(tasks, config_path=submission / "eval.yaml") + bundled = yaml.safe_load((task / "tests" / "eval.yaml").read_text()) + assert bundled["reward"]["score_range"] == [0, 1] + + def test_enrich_preserves_explicit_reward_section(tmp_path: Path): submission = tmp_path / "submission" (submission / "cases" / "case-001").mkdir(parents=True) From f06be06a89e335862348bae6d0bc4aebe20e5c3f Mon Sep 17 00:00:00 2001 From: gziv Date: Sun, 2 Aug 2026 11:11:11 +0300 Subject: [PATCH 05/12] chore(mlflow): retarget deploy defaults to ab-eval-flow Replace personal guy-ziv-evalflow namespace hardcoding in MLflow manifests and trigger_test_runs.sh defaults with the shared cluster namespace used by the rest of config/. --- config/mlflow/deployment.yaml | 3 ++- config/mlflow/pvc.yaml | 7 ++++--- config/mlflow/route.yaml | 4 ++-- config/mlflow/service.yaml | 2 +- scripts/misc/trigger_test_runs.sh | 9 +++++---- 5 files changed, 14 insertions(+), 11 deletions(-) diff --git a/config/mlflow/deployment.yaml b/config/mlflow/deployment.yaml index bdcba1ff..9385d06a 100644 --- a/config/mlflow/deployment.yaml +++ b/config/mlflow/deployment.yaml @@ -1,10 +1,11 @@ # Lightweight dedicated MLflow tracking server for ABEvalFlow AEH runs. # Backend: sqlite on PVC. Artifacts: same PVC under /mlflow/artifacts. +# Cluster DNS: http://abevalflow-mlflow.ab-eval-flow.svc.cluster.local:5000 apiVersion: apps/v1 kind: Deployment metadata: name: abevalflow-mlflow - namespace: guy-ziv-evalflow + namespace: ab-eval-flow labels: app.kubernetes.io/part-of: abevalflow app.kubernetes.io/name: mlflow diff --git a/config/mlflow/pvc.yaml b/config/mlflow/pvc.yaml index ab019001..de313f5d 100644 --- a/config/mlflow/pvc.yaml +++ b/config/mlflow/pvc.yaml @@ -1,10 +1,11 @@ -# Dedicated MLflow artifact + sqlite store for ABEvalFlow (dev). -# First deploy target: guy-ziv-evalflow. Promote to ab-eval-flow later if needed. +# Dedicated MLflow artifact + sqlite store for ABEvalFlow. +# Apply into the target namespace (default matches shared cluster config): +# oc apply -n ab-eval-flow -f config/mlflow/ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: abevalflow-mlflow-data - namespace: guy-ziv-evalflow + namespace: ab-eval-flow labels: app.kubernetes.io/part-of: abevalflow app.kubernetes.io/name: mlflow diff --git a/config/mlflow/route.yaml b/config/mlflow/route.yaml index 625c948f..8938673c 100644 --- a/config/mlflow/route.yaml +++ b/config/mlflow/route.yaml @@ -1,9 +1,9 @@ -# Optional external UI / API access for guy-ziv-evalflow MLflow. +# Optional external UI / API access for the shared-namespace MLflow. apiVersion: route.openshift.io/v1 kind: Route metadata: name: abevalflow-mlflow - namespace: guy-ziv-evalflow + namespace: ab-eval-flow labels: app.kubernetes.io/part-of: abevalflow app.kubernetes.io/name: mlflow diff --git a/config/mlflow/service.yaml b/config/mlflow/service.yaml index dba1870e..ebf3df81 100644 --- a/config/mlflow/service.yaml +++ b/config/mlflow/service.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: name: abevalflow-mlflow - namespace: guy-ziv-evalflow + namespace: ab-eval-flow labels: app.kubernetes.io/part-of: abevalflow app.kubernetes.io/name: mlflow diff --git a/scripts/misc/trigger_test_runs.sh b/scripts/misc/trigger_test_runs.sh index 3251016a..34057252 100755 --- a/scripts/misc/trigger_test_runs.sh +++ b/scripts/misc/trigger_test_runs.sh @@ -13,7 +13,7 @@ set -euo pipefail # legacy — original 6 Harbor/A2A/ASE runs only # # Env overrides: -# NAMESPACE OpenShift namespace (default: guy-ziv-evalflow) +# NAMESPACE OpenShift namespace (default: ab-eval-flow) # PIPELINE_CI CI pipeline name (default: abevalflow-pipeline-dev) # PIPELINE_MONITOR Monitoring pipeline name (default: abevalflow-monitoring-pipeline-dev) # LLM_API_BASE LiteLLM base URL @@ -21,6 +21,7 @@ set -euo pipefail # SKILL_REPO skill-submissions git URL # ENABLE_MLFLOW true/false (default false) # MLFLOW_TRACKING_URI MLflow server URI when ENABLE_MLFLOW=true +# (default: in-cluster abevalflow-mlflow Service) # # Verified AEH samples (skill-submissions): # single: revision=eval/aeh-hello-world-single submission-dir=aeh-hello-world-single @@ -32,7 +33,7 @@ if [[ -z "$BRANCH" ]]; then fi MODE="${2:-all}" -NAMESPACE="${NAMESPACE:-guy-ziv-evalflow}" +NAMESPACE="${NAMESPACE:-ab-eval-flow}" PIPELINE_CI="${PIPELINE_CI:-abevalflow-pipeline-dev}" PIPELINE_MONITOR="${PIPELINE_MONITOR:-abevalflow-monitoring-pipeline-dev}" LLM_API_BASE="${LLM_API_BASE:-http://litellm.ab-eval-flow.svc.cluster.local:4000}" @@ -40,8 +41,8 @@ AEH_IMAGE="${AEH_IMAGE:-quay.io/ecosystem-appeng/agent-eval-harness:v1.0.3}" SKILL_REPO="${SKILL_REPO:-https://github.com/RHEcosystemAppEng/skill-submissions.git}" LLM_MODEL="${LLM_MODEL:-claude-sonnet}" ENABLE_MLFLOW="${ENABLE_MLFLOW:-false}" -# Dedicated ABEvalFlow MLflow in guy-ziv-evalflow (see config/mlflow/). -MLFLOW_TRACKING_URI="${MLFLOW_TRACKING_URI:-http://abevalflow-mlflow.guy-ziv-evalflow.svc.cluster.local:5000}" +# In-cluster Service from config/mlflow/ (override for a personal namespace). +MLFLOW_TRACKING_URI="${MLFLOW_TRACKING_URI:-http://abevalflow-mlflow.ab-eval-flow.svc.cluster.local:5000}" echo "pipeline-repo-revision: $BRANCH" echo "namespace: $NAMESPACE" From 7573fbf222212f73571cc267427fb74c69e35bcf Mon Sep 17 00:00:00 2001 From: gziv Date: Sun, 2 Aug 2026 11:24:08 +0300 Subject: [PATCH 06/12] feat(aeh): add advisory regression, eval-check, and fuller MLflow hooks Wire non-blocking AEH score regression and harness inventory into evaluate/test, and expand MLflow logging to push-feedback plus optional sync-dataset. --- pipeline/pipelines/ci-pipeline-dev.yaml | 20 ++ pipeline/pipelines/ci-pipeline.yaml | 20 ++ .../pipelines/monitoring-pipeline-dev.yaml | 12 + pipeline/pipelines/monitoring-pipeline.yaml | 12 + pipeline/tasks/phases/evaluate.yaml | 50 +++++ pipeline/tasks/phases/test.yaml | 39 ++++ scripts/log_aeh_mlflow.py | 207 ++++++++++++++++-- scripts/run_aeh_eval_check.py | 124 +++++++++++ scripts/run_aeh_regression.py | 141 ++++++++++++ tests/test_log_aeh_mlflow.py | 24 ++ tests/test_run_aeh_eval_check.py | 44 ++++ tests/test_run_aeh_regression.py | 111 ++++++++++ 12 files changed, 782 insertions(+), 22 deletions(-) create mode 100644 scripts/run_aeh_eval_check.py create mode 100644 scripts/run_aeh_regression.py create mode 100644 tests/test_run_aeh_eval_check.py create mode 100644 tests/test_run_aeh_regression.py diff --git a/pipeline/pipelines/ci-pipeline-dev.yaml b/pipeline/pipelines/ci-pipeline-dev.yaml index ccc566dc..35ff695d 100644 --- a/pipeline/pipelines/ci-pipeline-dev.yaml +++ b/pipeline/pipelines/ci-pipeline-dev.yaml @@ -177,6 +177,18 @@ spec: type: string default: "" description: MLflow tracking server URI (e.g. http://mlflow.ab-eval-flow.svc:5000) + - name: enable-aeh-regression-check + type: string + default: "true" + description: Advisory AEH score.py regression after eval (does not fail the run) + - name: aeh-baseline-run-id + type: string + default: "" + description: Optional prior AEH run-id for regression --baseline + - name: enable-aeh-eval-check + type: string + default: "true" + description: Advisory AEH harness inventory in test phase (does not fail the run) # Security scanning - name: security-scan type: string @@ -300,6 +312,10 @@ spec: value: $(tasks.prepare.results.security-scan-use-llm) - name: enable-quality-review value: $(params.enable-test-quality-review) + - name: enable-aeh-eval-check + value: $(params.enable-aeh-eval-check) + - name: aeh-image + value: $(params.aeh-image) - name: llm-base-url value: $(params.llm-base-url) - name: llm-model @@ -407,6 +423,10 @@ spec: value: $(params.enable-mlflow) - name: mlflow-tracking-uri value: $(params.mlflow-tracking-uri) + - name: enable-aeh-regression-check + value: $(params.enable-aeh-regression-check) + - name: aeh-baseline-run-id + value: $(params.aeh-baseline-run-id) workspaces: - name: source workspace: shared-workspace diff --git a/pipeline/pipelines/ci-pipeline.yaml b/pipeline/pipelines/ci-pipeline.yaml index 56aee7d1..48532d86 100644 --- a/pipeline/pipelines/ci-pipeline.yaml +++ b/pipeline/pipelines/ci-pipeline.yaml @@ -177,6 +177,18 @@ spec: type: string default: "" description: MLflow tracking server URI (e.g. http://mlflow.ab-eval-flow.svc:5000) + - name: enable-aeh-regression-check + type: string + default: "true" + description: Advisory AEH score.py regression after eval (does not fail the run) + - name: aeh-baseline-run-id + type: string + default: "" + description: Optional prior AEH run-id for regression --baseline + - name: enable-aeh-eval-check + type: string + default: "true" + description: Advisory AEH harness inventory in test phase (does not fail the run) # Security scanning - name: security-scan type: string @@ -300,6 +312,10 @@ spec: value: $(tasks.prepare.results.security-scan-use-llm) - name: enable-quality-review value: $(params.enable-test-quality-review) + - name: enable-aeh-eval-check + value: $(params.enable-aeh-eval-check) + - name: aeh-image + value: $(params.aeh-image) - name: llm-base-url value: $(params.llm-base-url) - name: llm-model @@ -407,6 +423,10 @@ spec: value: $(params.enable-mlflow) - name: mlflow-tracking-uri value: $(params.mlflow-tracking-uri) + - name: enable-aeh-regression-check + value: $(params.enable-aeh-regression-check) + - name: aeh-baseline-run-id + value: $(params.aeh-baseline-run-id) workspaces: - name: source workspace: shared-workspace diff --git a/pipeline/pipelines/monitoring-pipeline-dev.yaml b/pipeline/pipelines/monitoring-pipeline-dev.yaml index 497aa56e..d8c2eff2 100644 --- a/pipeline/pipelines/monitoring-pipeline-dev.yaml +++ b/pipeline/pipelines/monitoring-pipeline-dev.yaml @@ -152,6 +152,14 @@ spec: type: string default: "" description: MLflow tracking server URI (e.g. http://mlflow.ab-eval-flow.svc:5000) + - name: enable-aeh-regression-check + type: string + default: "true" + description: Advisory AEH score.py regression after eval (does not fail the run) + - name: aeh-baseline-run-id + type: string + default: "" + description: Optional prior AEH run-id for regression --baseline # Thresholds - name: uplift-threshold type: string @@ -334,6 +342,10 @@ spec: value: $(params.enable-mlflow) - name: mlflow-tracking-uri value: $(params.mlflow-tracking-uri) + - name: enable-aeh-regression-check + value: $(params.enable-aeh-regression-check) + - name: aeh-baseline-run-id + value: $(params.aeh-baseline-run-id) workspaces: - name: source workspace: shared-workspace diff --git a/pipeline/pipelines/monitoring-pipeline.yaml b/pipeline/pipelines/monitoring-pipeline.yaml index 8edfb24c..473dfe60 100644 --- a/pipeline/pipelines/monitoring-pipeline.yaml +++ b/pipeline/pipelines/monitoring-pipeline.yaml @@ -152,6 +152,14 @@ spec: type: string default: "" description: MLflow tracking server URI (e.g. http://mlflow.ab-eval-flow.svc:5000) + - name: enable-aeh-regression-check + type: string + default: "true" + description: Advisory AEH score.py regression after eval (does not fail the run) + - name: aeh-baseline-run-id + type: string + default: "" + description: Optional prior AEH run-id for regression --baseline # Thresholds - name: uplift-threshold type: string @@ -334,6 +342,10 @@ spec: value: $(params.enable-mlflow) - name: mlflow-tracking-uri value: $(params.mlflow-tracking-uri) + - name: enable-aeh-regression-check + value: $(params.enable-aeh-regression-check) + - name: aeh-baseline-run-id + value: $(params.aeh-baseline-run-id) workspaces: - name: source workspace: shared-workspace diff --git a/pipeline/tasks/phases/evaluate.yaml b/pipeline/tasks/phases/evaluate.yaml index 4ea164e6..887af107 100644 --- a/pipeline/tasks/phases/evaluate.yaml +++ b/pipeline/tasks/phases/evaluate.yaml @@ -167,6 +167,18 @@ spec: type: string default: "" description: MLflow tracking server URI + - name: enable-aeh-regression-check + type: string + default: "true" + description: >- + When true, run AEH score.py regression after eval (advisory — does not + fail the step). Requires thresholds: in eval.yaml. + - name: aeh-baseline-run-id + type: string + default: "" + description: >- + Optional prior AEH run-id for score.py regression --baseline (relative + drift). Empty skips the baseline comparison. workspaces: - name: source description: Workspace containing the cloned submissions repository @@ -1460,6 +1472,25 @@ spec: echo "Skipping AEH MLflow (enable-mlflow=$(params.enable-mlflow), uri set=$([ -n "$(params.mlflow-tracking-uri)" ] && echo yes || echo no))" fi + # Advisory AEH threshold / baseline regression (never fails this step). + if [ "$(params.enable-aeh-regression-check)" = "true" ]; then + echo "=== AEH regression check (advisory) ===" + BASELINE_ARGS=() + if [ -n "$(params.aeh-baseline-run-id)" ]; then + BASELINE_ARGS=(--baseline "$(params.aeh-baseline-run-id)") + fi + python "$PIPELINE_DIR/scripts/run_aeh_regression.py" \ + --enabled true \ + --run-id "$RUN_ID" \ + --config "$SUBMISSION_DIR/eval.yaml" \ + --runs-dir "$(workspaces.source.path)/reports" \ + --output "$(workspaces.source.path)/reports/${SUBMISSION_NAME}/aeh-regression.txt" \ + "${BASELINE_ARGS[@]}" \ + 2>&1 || echo "Warning: AEH regression check failed to run (non-blocking)" + else + echo "Skipping AEH regression check (enable-aeh-regression-check=$(params.enable-aeh-regression-check))" + fi + if [ "$AEH_EXIT" -ne 0 ]; then echo "ERROR: AEH single evaluation exited $AEH_EXIT (report written if outputs existed)" exit "$AEH_EXIT" @@ -1731,6 +1762,25 @@ spec: echo "Skipping AEH MLflow (enable-mlflow=$(params.enable-mlflow))" fi + # Advisory AEH threshold / baseline regression on treatment run. + if [ "$(params.enable-aeh-regression-check)" = "true" ]; then + echo "=== AEH regression check (advisory, pairwise treatment) ===" + BASELINE_ARGS=() + if [ -n "$(params.aeh-baseline-run-id)" ]; then + BASELINE_ARGS=(--baseline "$(params.aeh-baseline-run-id)") + fi + python "$PIPELINE_DIR/scripts/run_aeh_regression.py" \ + --enabled true \ + --run-id "treatment-$RUN_ID" \ + --config "$SUBMISSION_DIR/$TREATMENT_CONFIG" \ + --runs-dir "$REPORTS_DIR" \ + --output "$REPORTS_DIR/${SUBMISSION_NAME}/aeh-regression.txt" \ + "${BASELINE_ARGS[@]}" \ + 2>&1 || echo "Warning: AEH regression check failed to run (non-blocking)" + else + echo "Skipping AEH regression check (enable-aeh-regression-check=$(params.enable-aeh-regression-check))" + fi + if [ "$PAIRWISE_EXIT" -ne 0 ]; then echo "ERROR: pairwise evaluation exited $PAIRWISE_EXIT (report written if outputs existed)" exit "$PAIRWISE_EXIT" diff --git a/pipeline/tasks/phases/test.yaml b/pipeline/tasks/phases/test.yaml index ae06fb64..edc45cb1 100644 --- a/pipeline/tasks/phases/test.yaml +++ b/pipeline/tasks/phases/test.yaml @@ -51,6 +51,16 @@ spec: type: string default: "false" description: Submission's skip_quality_review setting from metadata.yaml + - name: enable-aeh-eval-check + type: string + default: "true" + description: >- + When true and eval-engine=aeh, run AEH harness inventory (advisory — + does not fail the test phase). + - name: aeh-image + type: string + default: "quay.io/ecosystem-appeng/agent-eval-harness:v1.0.3" + description: AEH image providing skills/eval-check/scripts/harness_inventory.py - name: llm-base-url type: string default: "http://litellm.ab-eval-flow.svc:4000/v1" @@ -460,6 +470,35 @@ spec: echo "Quality review: passed=$PASSED recommendation=$REC" fi + # Step 5b: AEH harness inventory (advisory eval-check Step 1) + - name: aeh-eval-check + when: + - input: "$(params.eval-engine)" + operator: in + values: ["aeh"] + image: $(params.aeh-image) + script: | + #!/usr/bin/env bash + set -euo pipefail + echo "=== TEST PHASE: AEH eval-check (advisory inventory) ===" + + if [ "$(params.enable-aeh-eval-check)" != "true" ]; then + echo "Skipping AEH eval-check (enable-aeh-eval-check=$(params.enable-aeh-eval-check))" + exit 0 + fi + + PIPELINE_DIR="$(workspaces.source.path)/_pipeline" + SUBMISSION_PATH="$(workspaces.source.path)/submissions/$(params.submission-dir)" + OUT_DIR="$(workspaces.source.path)/reports/$(params.submission-name)" + mkdir -p "$OUT_DIR" + + export PYTHONPATH="$PIPELINE_DIR:${PYTHONPATH:-}" + python "$PIPELINE_DIR/scripts/run_aeh_eval_check.py" \ + --enabled true \ + --root "$SUBMISSION_PATH" \ + --output "$OUT_DIR/harness-inventory.yaml" \ + 2>&1 || echo "Warning: AEH eval-check failed to run (non-blocking)" + # Step 6: Finalize results and write metrics checkpoint - name: finalize image: registry.access.redhat.com/ubi9/python-311:9.6 diff --git a/scripts/log_aeh_mlflow.py b/scripts/log_aeh_mlflow.py index 80063782..2fa6a6ca 100644 --- a/scripts/log_aeh_mlflow.py +++ b/scripts/log_aeh_mlflow.py @@ -4,13 +4,19 @@ (under ``/opt/agent-eval-harness`` or ``AGENT_EVAL_HARNESS_ROOT``). Falls back to a minimal metrics/params logger from ``run_result.json`` + ``summary.yaml``. +Optional actions (same AEH skill tree): + - ``push-feedback`` — attach judge feedback to traces (``attach_feedback.py``) + - ``sync-dataset`` — sync cases to the MLflow dataset registry when a schema + mapping is available or a simple ``input.yaml:prompt`` default applies + Usage:: python scripts/log_aeh_mlflow.py \\ --run-id > \\ --config /path/to/eval.yaml \\ --runs-dir /workspace/source/reports \\ - --tracking-uri http://mlflow.example:5000 + --tracking-uri http://mlflow.example:5000 \\ + [--actions log-results,push-feedback,sync-dataset] Skip (exit 0) when ``--enabled false`` or tracking URI is empty. """ @@ -23,27 +29,145 @@ import os import subprocess import sys +import tempfile from pathlib import Path import yaml logger = logging.getLogger(__name__) -_AEH_LOG_RESULTS_CANDIDATES = (Path("/opt/agent-eval-harness/skills/eval-mlflow/scripts/log_results.py"),) +_AEH_MLFLOW_SCRIPTS = Path("/opt/agent-eval-harness/skills/eval-mlflow/scripts") +_AEH_LOG_RESULTS_CANDIDATES = (_AEH_MLFLOW_SCRIPTS / "log_results.py",) +_DEFAULT_ACTIONS = ("log-results", "push-feedback", "sync-dataset") -def _resolve_log_results_script() -> Path | None: +def _aeh_script(name: str) -> Path | None: root = os.environ.get("AGENT_EVAL_HARNESS_ROOT", "").strip() candidates: list[Path] = [] if root: - candidates.append(Path(root) / "skills/eval-mlflow/scripts/log_results.py") - candidates.extend(_AEH_LOG_RESULTS_CANDIDATES) + candidates.append(Path(root) / "skills/eval-mlflow/scripts" / name) + candidates.append(_AEH_MLFLOW_SCRIPTS / name) for path in candidates: if path.is_file(): return path return None +def _resolve_log_results_script() -> Path | None: + return _aeh_script("log_results.py") + + +def _parse_actions(raw: str) -> list[str]: + actions = [a.strip() for a in (raw or "").split(",") if a.strip()] + return actions or list(_DEFAULT_ACTIONS) + + +def _default_schema_mapping(config: Path) -> dict | None: + """Build a minimal mapping when cases expose ``input.yaml`` with ``prompt``.""" + try: + raw = yaml.safe_load(config.read_text()) or {} + except (OSError, yaml.YAMLError): + return None + dataset = raw.get("dataset") if isinstance(raw.get("dataset"), dict) else {} + rel = str(dataset.get("path") or "cases") + cases_dir = (config.parent / rel).resolve() + if not cases_dir.is_dir(): + return None + for case in sorted(p for p in cases_dir.iterdir() if p.is_dir()): + input_yaml = case / "input.yaml" + if not input_yaml.is_file(): + continue + try: + data = yaml.safe_load(input_yaml.read_text()) or {} + except (OSError, yaml.YAMLError): + continue + if isinstance(data, dict) and "prompt" in data: + mapping: dict = {"inputs": {"prompt": "input.yaml:prompt"}, "expectations": {}} + # Optional gold files commonly used by AEH samples + for name in ("reference.md", "reference.yaml"): + if (case / name).is_file(): + mapping["expectations"]["reference"] = f"{name}:__file__" + break + return mapping + return None + + +def _run_aeh_script(script: Path, argv: list[str], *, runs_dir: Path) -> int: + env = os.environ.copy() + env["AGENT_EVAL_RUNS_DIR"] = str(runs_dir) + cmd = [sys.executable, str(script), *argv] + logger.info("Running AEH MLflow script: %s", " ".join(cmd)) + result = subprocess.run(cmd, env=env, check=False, capture_output=True, text=True) + if result.stdout: + print(result.stdout, end="") + if result.stderr: + print(result.stderr, end="", file=sys.stderr) + return result.returncode + + +def _maybe_sync_dataset(*, config: Path, runs_dir: Path) -> None: + script = _aeh_script("sync_dataset.py") + if script is None: + logger.info("sync_dataset.py not found; skipping sync-dataset") + return + + mapping_path = config.parent / "schema_mapping.json" + tmp_path: Path | None = None + if mapping_path.is_file(): + logger.info("Using schema mapping: %s", mapping_path) + else: + mapping = _default_schema_mapping(config) + if mapping is None: + logger.info( + "No schema_mapping.json and no default input.yaml:prompt mapping; " + "skipping sync-dataset" + ) + return + tmp = tempfile.NamedTemporaryFile( + mode="w", suffix=".json", delete=False, prefix="aeh-schema-mapping-" + ) + json.dump(mapping, tmp) + tmp.close() + tmp_path = Path(tmp.name) + mapping_path = tmp_path + logger.info("Using auto-generated schema mapping for sync-dataset") + + try: + rc = _run_aeh_script( + script, + ["--config", str(config), "--mapping", str(mapping_path)], + runs_dir=runs_dir, + ) + if rc != 0: + logger.warning("sync-dataset exited %s (non-blocking)", rc) + finally: + if tmp_path is not None: + tmp_path.unlink(missing_ok=True) + + +def _maybe_push_feedback(*, run_id: str, config: Path, runs_dir: Path) -> None: + script = _aeh_script("attach_feedback.py") + if script is None: + logger.info("attach_feedback.py not found; skipping push-feedback") + return + rc = _run_aeh_script( + script, + [ + "--run-id", + run_id, + "--config", + str(config), + "--action", + "push", + "--source", + "judge", + ], + runs_dir=runs_dir, + ) + if rc != 0: + logger.warning("push-feedback exited %s (non-blocking)", rc) + + def _mlflow_importable() -> bool: try: import mlflow # noqa: F401 @@ -82,6 +206,33 @@ def _run_aeh_log_results(*, script: Path, run_id: str, config: Path, runs_dir: P return result.returncode +def _do_log_results(*, run_id: str, config: Path, runs_dir: Path, tracking_uri: str) -> int: + """Log results via AEH script or minimal fallback. Returns 0 on success.""" + script = _resolve_log_results_script() + if script is not None: + rc = _run_aeh_log_results( + script=script, + run_id=run_id, + config=config, + runs_dir=runs_dir, + ) + if rc == 0: + return 0 + logger.warning("AEH log_results.py exited %s; trying minimal logger", rc) + + try: + _minimal_mlflow_log( + run_id=run_id, + config=config, + runs_dir=runs_dir, + tracking_uri=tracking_uri, + ) + except Exception: + logger.exception("MLflow logging failed") + return 1 + return 0 + + def _minimal_mlflow_log(*, run_id: str, config: Path, runs_dir: Path, tracking_uri: str) -> None: try: import mlflow @@ -164,6 +315,14 @@ def main(argv: list[str] | None = None) -> int: default="true", help="Set to false/0/no to skip (default: true when URI set by caller)", ) + parser.add_argument( + "--actions", + default=",".join(_DEFAULT_ACTIONS), + help=( + "Comma-separated actions: log-results, push-feedback, sync-dataset " + f"(default: {','.join(_DEFAULT_ACTIONS)})" + ), + ) args = parser.parse_args(argv) enabled = str(args.enabled).strip().lower() in {"1", "true", "yes", "on"} @@ -188,6 +347,9 @@ def main(argv: list[str] | None = None) -> int: ) return 1 + actions = _parse_actions(args.actions) + logger.info("MLflow actions: %s", ", ".join(actions)) + # Belt-and-suspenders: AEH log_results rejects absolute harbor_job_dir. try: from abevalflow.harbor_extensions.aeh_paths import rewrite_harbor_job_dir @@ -202,29 +364,30 @@ def main(argv: list[str] | None = None) -> int: except Exception: logger.warning("harbor_job_dir rewrite failed; continuing", exc_info=True) - script = _resolve_log_results_script() - if script is not None: - rc = _run_aeh_log_results( - script=script, - run_id=args.run_id, - config=config, - runs_dir=runs_dir, - ) - if rc == 0: - return 0 - logger.warning("AEH log_results.py exited %s; trying minimal logger", rc) + # sync-dataset is run-independent; do it first so the registry is warm. + if "sync-dataset" in actions: + try: + _maybe_sync_dataset(config=config, runs_dir=runs_dir) + except Exception: + logger.warning("sync-dataset failed (non-blocking)", exc_info=True) - try: - _minimal_mlflow_log( + log_rc = 0 + if "log-results" in actions: + log_rc = _do_log_results( run_id=args.run_id, config=config, runs_dir=runs_dir, tracking_uri=tracking_uri, ) - except Exception: - logger.exception("MLflow logging failed") - return 1 - return 0 + + if "push-feedback" in actions: + try: + _maybe_push_feedback(run_id=args.run_id, config=config, runs_dir=runs_dir) + except Exception: + logger.warning("push-feedback failed (non-blocking)", exc_info=True) + + # Keep historical behavior: non-zero only when log-results itself fails. + return log_rc if __name__ == "__main__": diff --git a/scripts/run_aeh_eval_check.py b/scripts/run_aeh_eval_check.py new file mode 100644 index 00000000..4004d008 --- /dev/null +++ b/scripts/run_aeh_eval_check.py @@ -0,0 +1,124 @@ +#!/usr/bin/env python3 +"""Run AEH harness inventory (eval-check Step 1) in advisory mode. + +The full ``/eval-check`` skill also does LLM cross-component analysis; in Tekton +we only run the deterministic ``harness_inventory.py`` scanner and write a +report artifact. Always exits 0 unless ``--strict`` is set. + +Usage:: + + python scripts/run_aeh_eval_check.py \\ + --root /workspace/source/submissions/my-skill \\ + --output /workspace/source/reports/my-skill/harness-inventory.yaml +""" + +from __future__ import annotations + +import argparse +import logging +import os +import subprocess +import sys +from pathlib import Path + +logger = logging.getLogger(__name__) + +_INVENTORY_CANDIDATES = ( + Path("/opt/agent-eval-harness/skills/eval-check/scripts/harness_inventory.py"), +) + + +def _resolve_inventory_script() -> Path | None: + root = os.environ.get("AGENT_EVAL_HARNESS_ROOT", "").strip() + candidates: list[Path] = [] + if root: + candidates.append(Path(root) / "skills/eval-check/scripts/harness_inventory.py") + candidates.extend(_INVENTORY_CANDIDATES) + for path in candidates: + if path.is_file(): + return path + return None + + +def main(argv: list[str] | None = None) -> int: + logging.basicConfig(level=logging.INFO, format="%(message)s") + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "--root", + type=Path, + required=True, + help="Submission / project root to scan (skills/, CLAUDE.md, …)", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Where to write inventory YAML (default: stdout only)", + ) + parser.add_argument( + "--enabled", + default="true", + help="Set false/0/no to skip (default: true)", + ) + parser.add_argument( + "--strict", + action="store_true", + help="Exit non-zero if the inventory script fails (default: advisory)", + ) + args = parser.parse_args(argv) + + enabled = str(args.enabled).strip().lower() in {"1", "true", "yes", "on"} + if not enabled: + logger.info("AEH eval-check disabled (--enabled=%s)", args.enabled) + return 0 + + root = args.root.resolve() + if not root.is_dir(): + logger.warning("Eval-check root not found: %s — skipping", root) + return 0 + + script = _resolve_inventory_script() + if script is None: + logger.warning( + "AEH harness_inventory.py not found " + "(set AGENT_EVAL_HARNESS_ROOT or use AEH image); skipping" + ) + return 0 + + cmd = [sys.executable, str(script), "--root", str(root), "--format", "yaml"] + # Older inventory scripts may not support --format; fall back to text. + logger.info("Running AEH harness inventory (advisory): %s", " ".join(cmd)) + result = subprocess.run(cmd, check=False, capture_output=True, text=True) + if result.returncode != 0 and "unrecognized arguments" in (result.stderr or ""): + cmd = [sys.executable, str(script), "--root", str(root)] + logger.info("Retrying without --format: %s", " ".join(cmd)) + result = subprocess.run(cmd, check=False, capture_output=True, text=True) + + combined = (result.stdout or "") + (result.stderr or "") + if result.stdout: + print(result.stdout, end="") + if result.stderr: + print(result.stderr, end="", file=sys.stderr) + + if args.output is not None: + args.output.parent.mkdir(parents=True, exist_ok=True) + header = ( + "# AEH harness inventory (advisory — inventory only; " + "full /eval-check LLM analysis is interactive)\n" + f"# root={root}\n" + f"# exit_code={result.returncode}\n\n" + ) + args.output.write_text(header + combined) + logger.info("Wrote inventory report: %s", args.output) + + if result.returncode != 0: + logger.warning( + "AEH harness inventory exited %s — advisory, not failing the step", + result.returncode, + ) + return result.returncode if args.strict else 0 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/run_aeh_regression.py b/scripts/run_aeh_regression.py new file mode 100644 index 00000000..1d8eb033 --- /dev/null +++ b/scripts/run_aeh_regression.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python3 +"""Run AEH ``score.py regression`` in advisory (non-blocking) mode. + +Loads an already-scored AEH run's ``summary.yaml`` and compares judge +aggregates to ``thresholds:`` in ``eval.yaml``. Optionally compares against +a prior run via ``--baseline``. + +Always exits 0 unless ``--strict`` is set, so Tekton can surface the check +without failing the PipelineRun. + +Usage:: + + python scripts/run_aeh_regression.py \\ + --run-id \\ + --config /path/to/eval.yaml \\ + --runs-dir /workspace/source/reports \\ + [--baseline ] \\ + [--output /path/to/aeh-regression.txt] +""" + +from __future__ import annotations + +import argparse +import logging +import os +import subprocess +import sys +from pathlib import Path + +logger = logging.getLogger(__name__) + +_SCORE_CANDIDATES = ( + Path("/opt/agent-eval-harness/skills/eval-run/scripts/score.py"), +) + + +def _resolve_score_script() -> Path | None: + root = os.environ.get("AGENT_EVAL_HARNESS_ROOT", "").strip() + candidates: list[Path] = [] + if root: + candidates.append(Path(root) / "skills/eval-run/scripts/score.py") + candidates.extend(_SCORE_CANDIDATES) + for path in candidates: + if path.is_file(): + return path + return None + + +def main(argv: list[str] | None = None) -> int: + logging.basicConfig(level=logging.INFO, format="%(message)s") + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--run-id", required=True) + parser.add_argument("--config", type=Path, required=True) + parser.add_argument("--runs-dir", type=Path, required=True) + parser.add_argument( + "--baseline", + default="", + help="Optional prior AEH run-id under the same eval for relative drift check", + ) + parser.add_argument( + "--output", + type=Path, + default=None, + help="Optional path to write the regression report text", + ) + parser.add_argument( + "--enabled", + default="true", + help="Set false/0/no to skip (default: true)", + ) + parser.add_argument( + "--strict", + action="store_true", + help="Exit non-zero when AEH reports regressions (default: advisory)", + ) + args = parser.parse_args(argv) + + enabled = str(args.enabled).strip().lower() in {"1", "true", "yes", "on"} + if not enabled: + logger.info("AEH regression check disabled (--enabled=%s)", args.enabled) + return 0 + + config = args.config.resolve() + runs_dir = args.runs_dir.resolve() + if not config.is_file(): + logger.error("Config not found: %s", config) + return 0 if not args.strict else 1 + + script = _resolve_score_script() + if script is None: + logger.warning( + "AEH score.py not found (set AGENT_EVAL_HARNESS_ROOT or use AEH image); skipping" + ) + return 0 + + env = os.environ.copy() + env["AGENT_EVAL_RUNS_DIR"] = str(runs_dir) + cmd = [ + sys.executable, + str(script), + "regression", + "--run-id", + args.run_id, + "--config", + str(config), + ] + baseline = (args.baseline or "").strip() + if baseline: + cmd.extend(["--baseline", baseline]) + + logger.info("Running AEH regression (advisory): %s", " ".join(cmd)) + result = subprocess.run(cmd, env=env, check=False, capture_output=True, text=True) + combined = (result.stdout or "") + (result.stderr or "") + if result.stdout: + print(result.stdout, end="") + if result.stderr: + print(result.stderr, end="", file=sys.stderr) + + if args.output is not None: + args.output.parent.mkdir(parents=True, exist_ok=True) + header = ( + f"# AEH regression (advisory)\n" + f"# run_id={args.run_id}\n" + f"# baseline={baseline or '(none)'}\n" + f"# exit_code={result.returncode}\n\n" + ) + args.output.write_text(header + combined) + + if result.returncode == 0: + logger.info("AEH regression: REGRESSIONS: 0 (advisory)") + return 0 + + logger.warning( + "AEH regression reported issues (exit %s) — advisory, not failing the step", + result.returncode, + ) + return result.returncode if args.strict else 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_log_aeh_mlflow.py b/tests/test_log_aeh_mlflow.py index 33968f8f..bd008e02 100644 --- a/tests/test_log_aeh_mlflow.py +++ b/tests/test_log_aeh_mlflow.py @@ -102,6 +102,8 @@ def _block_mlflow(name, *args, **kwargs): "http://mlflow.example:5000", "--enabled", "true", + "--actions", + "log-results", ] ) assert rc == 1 @@ -143,6 +145,11 @@ def log_artifact(self, path): monkeypatch.setitem(sys.modules, "mlflow", _FakeMlflow()) + # Restrict to log-results so missing AEH push/sync scripts do not matter. + monkeypatch.setattr( + "scripts.log_aeh_mlflow._resolve_log_results_script", + lambda: None, + ) rc = main( [ "--run-id", @@ -155,6 +162,8 @@ def log_artifact(self, path): "http://mlflow.example:5000", "--enabled", "true", + "--actions", + "log-results", ] ) assert rc == 0 @@ -162,3 +171,18 @@ def log_artifact(self, path): assert calls["experiment"] == "aeh-demo" assert ("mean_reward", 0.8) in calls["metrics"] assert ("tokens_input", 100.0) in calls["metrics"] + + +def test_default_schema_mapping_from_input_yaml(tmp_path: Path) -> None: + from scripts.log_aeh_mlflow import _default_schema_mapping + + config = tmp_path / "eval.yaml" + cases = tmp_path / "cases" / "case-001" + cases.mkdir(parents=True) + (cases / "input.yaml").write_text(yaml.safe_dump({"prompt": "hello"})) + (cases / "reference.md").write_text("gold\n") + config.write_text(yaml.safe_dump({"skill": "demo", "dataset": {"path": "cases"}})) + mapping = _default_schema_mapping(config) + assert mapping is not None + assert mapping["inputs"]["prompt"] == "input.yaml:prompt" + assert mapping["expectations"]["reference"] == "reference.md:__file__" diff --git a/tests/test_run_aeh_eval_check.py b/tests/test_run_aeh_eval_check.py new file mode 100644 index 00000000..e51583cd --- /dev/null +++ b/tests/test_run_aeh_eval_check.py @@ -0,0 +1,44 @@ +"""Tests for scripts/run_aeh_eval_check.py.""" + +from __future__ import annotations + +from pathlib import Path + +from scripts.run_aeh_eval_check import main + + +def test_skips_when_disabled(tmp_path: Path) -> None: + root = tmp_path / "submission" + root.mkdir() + rc = main(["--root", str(root), "--enabled", "false"]) + assert rc == 0 + + +def test_skips_when_inventory_missing(tmp_path: Path, monkeypatch) -> None: + root = tmp_path / "submission" + root.mkdir() + monkeypatch.setattr( + "scripts.run_aeh_eval_check._resolve_inventory_script", + lambda: None, + ) + rc = main(["--root", str(root)]) + assert rc == 0 + + +def test_writes_report_and_stays_advisory(tmp_path: Path, monkeypatch) -> None: + root = tmp_path / "submission" + root.mkdir() + fake = tmp_path / "harness_inventory.py" + fake.write_text( + "import sys\n" + "print('skills: 1')\n" + "sys.exit(1)\n" + ) + monkeypatch.setattr( + "scripts.run_aeh_eval_check._resolve_inventory_script", + lambda: fake, + ) + out = tmp_path / "inventory.yaml" + rc = main(["--root", str(root), "--output", str(out)]) + assert rc == 0 + assert "skills: 1" in out.read_text() diff --git a/tests/test_run_aeh_regression.py b/tests/test_run_aeh_regression.py new file mode 100644 index 00000000..76b6dcc7 --- /dev/null +++ b/tests/test_run_aeh_regression.py @@ -0,0 +1,111 @@ +"""Tests for scripts/run_aeh_regression.py.""" + +from __future__ import annotations + +from pathlib import Path + +import yaml + +from scripts.run_aeh_regression import main + + +def _write_config(tmp_path: Path) -> Path: + config = tmp_path / "eval.yaml" + config.write_text( + yaml.safe_dump( + { + "skill": "demo", + "thresholds": {"correctness": {"min_mean": 0.7}}, + "judges": [{"name": "correctness", "check": "return True, 'ok'"}], + } + ) + ) + return config + + +def test_skips_when_disabled(tmp_path: Path) -> None: + config = _write_config(tmp_path) + rc = main( + [ + "--run-id", + "r1", + "--config", + str(config), + "--runs-dir", + str(tmp_path / "reports"), + "--enabled", + "false", + ] + ) + assert rc == 0 + + +def test_skips_when_score_py_missing(tmp_path: Path, monkeypatch) -> None: + config = _write_config(tmp_path) + monkeypatch.delenv("AGENT_EVAL_HARNESS_ROOT", raising=False) + monkeypatch.setattr( + "scripts.run_aeh_regression._resolve_score_script", + lambda: None, + ) + rc = main( + [ + "--run-id", + "r1", + "--config", + str(config), + "--runs-dir", + str(tmp_path / "reports"), + ] + ) + assert rc == 0 + + +def test_advisory_exit_zero_when_score_fails(tmp_path: Path, monkeypatch) -> None: + config = _write_config(tmp_path) + fake = tmp_path / "score.py" + fake.write_text("import sys; print('REGRESSIONS: 1 detected'); sys.exit(1)\n") + monkeypatch.setattr( + "scripts.run_aeh_regression._resolve_score_script", + lambda: fake, + ) + out = tmp_path / "regression.txt" + rc = main( + [ + "--run-id", + "r1", + "--config", + str(config), + "--runs-dir", + str(tmp_path / "reports"), + "--output", + str(out), + "--baseline", + "prior-run", + ] + ) + assert rc == 0 + text = out.read_text() + assert "REGRESSIONS: 1 detected" in text + assert "baseline=prior-run" in text + + +def test_strict_propagates_failure(tmp_path: Path, monkeypatch) -> None: + config = _write_config(tmp_path) + fake = tmp_path / "score.py" + fake.write_text("import sys; sys.exit(1)\n") + monkeypatch.setattr( + "scripts.run_aeh_regression._resolve_score_script", + lambda: fake, + ) + rc = main( + [ + "--run-id", + "r1", + "--config", + str(config), + "--runs-dir", + str(tmp_path / "reports"), + "--strict", + ] + ) + assert rc == 1 From 8ae9ddb847fda688a217e8e31904a8998c3f82b3 Mon Sep 17 00:00:00 2001 From: gziv Date: Sun, 2 Aug 2026 11:40:14 +0300 Subject: [PATCH 07/12] fix(aeh): address PR #61 review blockers Format with ruff, resolve pairwise MLflow run dirs, tighten MLflow host allowlist, document optional Route, and only set score_range on injected rewards. --- .../harbor_extensions/aeh_task_enrichment.py | 19 +++------- config/mlflow/README.md | 32 ++++++++++++++++ config/mlflow/deployment.yaml | 6 ++- config/mlflow/route.yaml | 15 +++++++- scripts/log_aeh_mlflow.py | 37 +++++++++++-------- scripts/run_aeh_eval_check.py | 9 +---- scripts/run_aeh_regression.py | 8 +--- tests/test_aeh_task_enrichment.py | 9 ++++- tests/test_log_aeh_mlflow.py | 26 +++++++++++++ tests/test_run_aeh_eval_check.py | 6 +-- 10 files changed, 114 insertions(+), 53 deletions(-) create mode 100644 config/mlflow/README.md diff --git a/abevalflow/harbor_extensions/aeh_task_enrichment.py b/abevalflow/harbor_extensions/aeh_task_enrichment.py index d8f7003a..ca801599 100644 --- a/abevalflow/harbor_extensions/aeh_task_enrichment.py +++ b/abevalflow/harbor_extensions/aeh_task_enrichment.py @@ -112,11 +112,7 @@ def _reward_to_inject(raw: dict) -> dict | None: judges = raw.get("judges") or [] if not isinstance(judges, list): return None - names = [ - str(j["name"]) - for j in judges - if isinstance(j, dict) and j.get("name") - ] + names = [str(j["name"]) for j in judges if isinstance(j, dict) and j.get("name")] if not names: return None return { @@ -185,16 +181,11 @@ def _enrich_one_task( models["skill"] = skill_model if judge_model: models["judge"] = judge_model - if reward_inject is not None and not ( - isinstance(cfg.get("reward"), dict) and cfg["reward"] - ): + if reward_inject is not None and not (isinstance(cfg.get("reward"), dict) and cfg["reward"]): + # Only ABEvalFlow-injected rewards get score_range [0, 1]. Do not + # rewrite submission-authored weighted rewards that omit score_range + # (AEH default [1, 5] may be intentional for Likert judges). cfg["reward"] = reward_inject - # Harden existing reward blocks that omit score_range (AEH default [1,5] - # collapses [0,1] judge scores of 1.0 to reward 0.0). - reward = cfg.get("reward") - if isinstance(reward, dict) and reward.get("formula") == "weighted": - if "score_range" not in reward and "raw" not in reward: - reward["score_range"] = [0, 1] bundled.write_text(yaml.safe_dump(cfg, sort_keys=False, allow_unicode=True)) # 3) Stage annotations in test.sh before reward runs diff --git a/config/mlflow/README.md b/config/mlflow/README.md new file mode 100644 index 00000000..fb41cacb --- /dev/null +++ b/config/mlflow/README.md @@ -0,0 +1,32 @@ +# ABEvalFlow MLflow (eval-only) + +Lightweight tracking server for AEH post-evaluate logging. **Not** production +telemetry HA — single replica, SQLite on a PVC. + +## Apply (in-cluster CI) + +```bash +oc apply -f config/mlflow/pvc.yaml +oc apply -f config/mlflow/service.yaml +oc apply -f config/mlflow/deployment.yaml +``` + +Pipeline tracking URI (ClusterIP): + +```text +http://abevalflow-mlflow.ab-eval-flow.svc.cluster.local:5000 +``` + +Set pipeline params `enable-mlflow=true` and `mlflow-tracking-uri` to that URL. + +## Security defaults + +- `--allowed-hosts` lists in-cluster Service DNS + localhost only (no `*`). +- CORS wildcards are **not** enabled; browser UIs need an explicit allowlist. +- `route.yaml` is **optional / dev-only**. Do not apply it unless you also + append the Route hostname to `--allowed-hosts` in `deployment.yaml`. + +## Storage + +`pvc.yaml` sets `storageClassName: gp3` (AWS). On other clusters, edit or omit +the field so the cluster default StorageClass applies. diff --git a/config/mlflow/deployment.yaml b/config/mlflow/deployment.yaml index 9385d06a..8dfe5d68 100644 --- a/config/mlflow/deployment.yaml +++ b/config/mlflow/deployment.yaml @@ -42,11 +42,13 @@ spec: # A filesystem default-artifact-root makes remote clients try to # mkdir /mlflow locally (PermissionError under OpenShift SCC). # Single uvicorn worker: default multi-worker forks OOM under small limits. + # Host allowlist: in-cluster Service DNS + localhost only. + # Do not use '*'. If you expose a Route, append its hostname here + # (see README.md). CORS is omitted — pipeline clients are not browsers. exec mlflow server \ --host 0.0.0.0 \ --port 5000 \ - --allowed-hosts '*' \ - --cors-allowed-origins '*' \ + --allowed-hosts 'abevalflow-mlflow,abevalflow-mlflow.ab-eval-flow,abevalflow-mlflow.ab-eval-flow.svc,abevalflow-mlflow.ab-eval-flow.svc.cluster.local,localhost,127.0.0.1' \ --backend-store-uri sqlite:////mlflow/mlflow.db \ --default-artifact-root mlflow-artifacts:/ \ --artifacts-destination /mlflow/artifacts \ diff --git a/config/mlflow/route.yaml b/config/mlflow/route.yaml index 8938673c..17e734c9 100644 --- a/config/mlflow/route.yaml +++ b/config/mlflow/route.yaml @@ -1,4 +1,15 @@ -# Optional external UI / API access for the shared-namespace MLflow. +# OPTIONAL — do not apply by default. +# +# Exposes abevalflow-mlflow outside the cluster. The Deployment host allowlist +# is in-cluster only; if you apply this Route you MUST append the Route +# hostname to --allowed-hosts in deployment.yaml. +# +# Intended for eval/dev UI access only — not production telemetry. Prefer +# ClusterIP + in-cluster MLFLOW_TRACKING_URI for PipelineRuns: +# http://abevalflow-mlflow.ab-eval-flow.svc.cluster.local:5000 +# +# Apply explicitly when needed: +# oc apply -f config/mlflow/route.yaml apiVersion: route.openshift.io/v1 kind: Route metadata: @@ -7,6 +18,8 @@ metadata: labels: app.kubernetes.io/part-of: abevalflow app.kubernetes.io/name: mlflow + annotations: + abevalflow.io/exposure: "optional-dev-only" spec: to: kind: Service diff --git a/scripts/log_aeh_mlflow.py b/scripts/log_aeh_mlflow.py index 2fa6a6ca..dd8aade7 100644 --- a/scripts/log_aeh_mlflow.py +++ b/scripts/log_aeh_mlflow.py @@ -118,14 +118,9 @@ def _maybe_sync_dataset(*, config: Path, runs_dir: Path) -> None: else: mapping = _default_schema_mapping(config) if mapping is None: - logger.info( - "No schema_mapping.json and no default input.yaml:prompt mapping; " - "skipping sync-dataset" - ) + logger.info("No schema_mapping.json and no default input.yaml:prompt mapping; skipping sync-dataset") return - tmp = tempfile.NamedTemporaryFile( - mode="w", suffix=".json", delete=False, prefix="aeh-schema-mapping-" - ) + tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False, prefix="aeh-schema-mapping-") json.dump(mapping, tmp) tmp.close() tmp_path = Path(tmp.name) @@ -233,6 +228,23 @@ def _do_log_results(*, run_id: str, config: Path, runs_dir: Path, tracking_uri: return 0 +def _resolve_run_dir(runs_dir: Path, skill: str, run_id: str) -> Path: + """Locate ``runs_dir//``, with pairwise prefix fallbacks. + + Pairwise AEH layouts use ``control-`` / ``treatment-``. When callers + pass a bare pipeline-run id, prefer the treatment dir, then control. + """ + candidates = [runs_dir / skill / run_id] + if not run_id.startswith(("control-", "treatment-")): + candidates.append(runs_dir / skill / f"treatment-{run_id}") + candidates.append(runs_dir / skill / f"control-{run_id}") + for candidate in candidates: + if candidate.is_dir(): + return candidate + tried = ", ".join(str(c) for c in candidates) + raise FileNotFoundError(f"AEH run dir not found for skill={skill!r} run_id={run_id!r}; tried: {tried}") + + def _minimal_mlflow_log(*, run_id: str, config: Path, runs_dir: Path, tracking_uri: str) -> None: try: import mlflow @@ -244,13 +256,7 @@ def _minimal_mlflow_log(*, run_id: str, config: Path, runs_dir: Path, tracking_u mlflow_cfg = raw.get("mlflow") if isinstance(raw.get("mlflow"), dict) else {} experiment = str(mlflow_cfg.get("experiment") or skill or "abevalflow-aeh") - run_dir = runs_dir / skill / run_id - if not run_dir.is_dir(): - # Pairwise control/treatment dirs use control- / treatment- - # when --run-id already includes the prefix. - run_dir = runs_dir / skill / run_id - if not run_dir.is_dir(): - raise FileNotFoundError(f"AEH run dir not found: {run_dir}") + run_dir = _resolve_run_dir(runs_dir, skill, run_id) run_result: dict = {} rr_path = run_dir / "run_result.json" @@ -319,8 +325,7 @@ def main(argv: list[str] | None = None) -> int: "--actions", default=",".join(_DEFAULT_ACTIONS), help=( - "Comma-separated actions: log-results, push-feedback, sync-dataset " - f"(default: {','.join(_DEFAULT_ACTIONS)})" + f"Comma-separated actions: log-results, push-feedback, sync-dataset (default: {','.join(_DEFAULT_ACTIONS)})" ), ) args = parser.parse_args(argv) diff --git a/scripts/run_aeh_eval_check.py b/scripts/run_aeh_eval_check.py index 4004d008..73bd6d79 100644 --- a/scripts/run_aeh_eval_check.py +++ b/scripts/run_aeh_eval_check.py @@ -23,9 +23,7 @@ logger = logging.getLogger(__name__) -_INVENTORY_CANDIDATES = ( - Path("/opt/agent-eval-harness/skills/eval-check/scripts/harness_inventory.py"), -) +_INVENTORY_CANDIDATES = (Path("/opt/agent-eval-harness/skills/eval-check/scripts/harness_inventory.py"),) def _resolve_inventory_script() -> Path | None: @@ -79,10 +77,7 @@ def main(argv: list[str] | None = None) -> int: script = _resolve_inventory_script() if script is None: - logger.warning( - "AEH harness_inventory.py not found " - "(set AGENT_EVAL_HARNESS_ROOT or use AEH image); skipping" - ) + logger.warning("AEH harness_inventory.py not found (set AGENT_EVAL_HARNESS_ROOT or use AEH image); skipping") return 0 cmd = [sys.executable, str(script), "--root", str(root), "--format", "yaml"] diff --git a/scripts/run_aeh_regression.py b/scripts/run_aeh_regression.py index 1d8eb033..8b5e750b 100644 --- a/scripts/run_aeh_regression.py +++ b/scripts/run_aeh_regression.py @@ -29,9 +29,7 @@ logger = logging.getLogger(__name__) -_SCORE_CANDIDATES = ( - Path("/opt/agent-eval-harness/skills/eval-run/scripts/score.py"), -) +_SCORE_CANDIDATES = (Path("/opt/agent-eval-harness/skills/eval-run/scripts/score.py"),) def _resolve_score_script() -> Path | None: @@ -88,9 +86,7 @@ def main(argv: list[str] | None = None) -> int: script = _resolve_score_script() if script is None: - logger.warning( - "AEH score.py not found (set AGENT_EVAL_HARNESS_ROOT or use AEH image); skipping" - ) + logger.warning("AEH score.py not found (set AGENT_EVAL_HARNESS_ROOT or use AEH image); skipping") return 0 env = os.environ.copy() diff --git a/tests/test_aeh_task_enrichment.py b/tests/test_aeh_task_enrichment.py index e9a4e7ee..c8b52071 100644 --- a/tests/test_aeh_task_enrichment.py +++ b/tests/test_aeh_task_enrichment.py @@ -216,7 +216,8 @@ def test_enrich_patches_models_and_injects_weighted_reward(tmp_path: Path): } -def test_enrich_adds_score_range_to_weighted_reward_without_one(tmp_path: Path): +def test_enrich_preserves_submission_reward_without_score_range(tmp_path: Path): + """Likert submissions may omit score_range (AEH default [1,5]); do not force [0,1].""" submission = tmp_path / "submission" (submission / "cases" / "case-001").mkdir(parents=True) (submission / "eval.yaml").write_text( @@ -252,7 +253,11 @@ def test_enrich_adds_score_range_to_weighted_reward_without_one(tmp_path: Path): enrich_harbor_tasks(tasks, config_path=submission / "eval.yaml") bundled = yaml.safe_load((task / "tests" / "eval.yaml").read_text()) - assert bundled["reward"]["score_range"] == [0, 1] + assert "score_range" not in bundled["reward"] + assert bundled["reward"]["weights"] == { + "correctness": 1.0, + "helpfulness": 1.0, + } def test_enrich_preserves_explicit_reward_section(tmp_path: Path): diff --git a/tests/test_log_aeh_mlflow.py b/tests/test_log_aeh_mlflow.py index bd008e02..881ab9a1 100644 --- a/tests/test_log_aeh_mlflow.py +++ b/tests/test_log_aeh_mlflow.py @@ -186,3 +186,29 @@ def test_default_schema_mapping_from_input_yaml(tmp_path: Path) -> None: assert mapping is not None assert mapping["inputs"]["prompt"] == "input.yaml:prompt" assert mapping["expectations"]["reference"] == "reference.md:__file__" + + +def test_resolve_run_dir_prefers_exact_then_pairwise_prefixes(tmp_path: Path) -> None: + from scripts.log_aeh_mlflow import _resolve_run_dir + + skill = "demo-skill" + runs = tmp_path / "reports" + exact = runs / skill / "run-1" + exact.mkdir(parents=True) + assert _resolve_run_dir(runs, skill, "run-1") == exact + + bare_id = "pipe-abc" + treatment = runs / skill / f"treatment-{bare_id}" + treatment.mkdir(parents=True) + assert _resolve_run_dir(runs, skill, bare_id) == treatment + + control_only = "pipe-ctrl" + control = runs / skill / f"control-{control_only}" + control.mkdir(parents=True) + assert _resolve_run_dir(runs, skill, control_only) == control + + try: + _resolve_run_dir(runs, skill, "missing-run") + raise AssertionError("expected FileNotFoundError") + except FileNotFoundError as exc: + assert "missing-run" in str(exc) diff --git a/tests/test_run_aeh_eval_check.py b/tests/test_run_aeh_eval_check.py index e51583cd..5c7d3344 100644 --- a/tests/test_run_aeh_eval_check.py +++ b/tests/test_run_aeh_eval_check.py @@ -29,11 +29,7 @@ def test_writes_report_and_stays_advisory(tmp_path: Path, monkeypatch) -> None: root = tmp_path / "submission" root.mkdir() fake = tmp_path / "harness_inventory.py" - fake.write_text( - "import sys\n" - "print('skills: 1')\n" - "sys.exit(1)\n" - ) + fake.write_text("import sys\nprint('skills: 1')\nsys.exit(1)\n") monkeypatch.setattr( "scripts.run_aeh_eval_check._resolve_inventory_script", lambda: fake, From d09724dddb5966acefe97724c4f153d20f99d36b Mon Sep 17 00:00:00 2001 From: gziv Date: Sun, 2 Aug 2026 11:46:56 +0300 Subject: [PATCH 08/12] fix(aeh): address PR #61 should-fix review items Bake mlflow into AEH image (keep /tmp pip fallback), harden MLflow no-op detection, default eval-check off, portable PVC, ops docs, missing job-dir rewrite. --- Docs/infrastructure_ops.md | 25 ++++++++++++++ abevalflow/harbor_extensions/aeh_paths.py | 16 +++++++-- config/mlflow/README.md | 9 +++-- config/mlflow/pvc.yaml | 4 ++- containers/agent-eval-harness/Containerfile | 6 +++- pipeline/pipelines/ci-pipeline-dev.yaml | 6 ++-- pipeline/pipelines/ci-pipeline.yaml | 6 ++-- pipeline/tasks/phases/evaluate.yaml | 17 +++++---- pipeline/tasks/phases/test.yaml | 5 +-- scripts/log_aeh_mlflow.py | 33 ++++++++++++++---- tests/test_aeh_paths.py | 14 ++++++++ tests/test_log_aeh_mlflow.py | 29 ++++++++++++++++ tests/test_validate_aeh.py | 38 +++++++++++++++++++++ 13 files changed, 182 insertions(+), 26 deletions(-) diff --git a/Docs/infrastructure_ops.md b/Docs/infrastructure_ops.md index 87619f83..e0216e75 100644 --- a/Docs/infrastructure_ops.md +++ b/Docs/infrastructure_ops.md @@ -109,6 +109,31 @@ curl http://localhost:4000/health Adjust sizes based on expected submission volume and image sizes. +## MLflow (optional AEH observability) + +Opt-in post-evaluate logging for AEH runs. Defaults: `enable-mlflow=false`, +empty `mlflow-tracking-uri`. See `config/mlflow/README.md` for security notes +(in-cluster hosts only; Route is optional/dev-only; SQLite is not HA). + +```bash +# Deploy tracking server (PVC uses cluster default StorageClass) +oc apply -f config/mlflow/pvc.yaml +oc apply -f config/mlflow/service.yaml +oc apply -f config/mlflow/deployment.yaml + +# Pipeline params (example) +# enable-mlflow: "true" +# mlflow-tracking-uri: http://abevalflow-mlflow.ab-eval-flow.svc.cluster.local:5000 +``` + +Do **not** apply `config/mlflow/route.yaml` unless you also append the Route +hostname to `--allowed-hosts` in `deployment.yaml`. + +The AEH runner image (`containers/agent-eval-harness/Containerfile`) bakes +`mlflow-skinny` + `pandas`. The evaluate step still pip-installs to `/tmp` only +when those imports are missing (older images). Rebuild/push the AEH image after +changing that Containerfile. + ## Cleanup CronJob Runs daily at 03:00 UTC. Configurable via environment variables: diff --git a/abevalflow/harbor_extensions/aeh_paths.py b/abevalflow/harbor_extensions/aeh_paths.py index a4287f7a..63bdc31c 100644 --- a/abevalflow/harbor_extensions/aeh_paths.py +++ b/abevalflow/harbor_extensions/aeh_paths.py @@ -16,6 +16,14 @@ def rewrite_harbor_job_dir(run_result_path: Path) -> str | None: resolves relative ones against ancestors of the run directory. Harbor ``run.py`` currently stores an absolute job dir, which skips trace build. + **Side effect:** mutates ``run_result.json`` in place (overwrites + ``harbor_job_dir``). Downstream consumers that need the original absolute + Harbor path should read it before calling this helper. + + When the job directory is missing (moved/archived), still attempt a + relative rewrite from the absolute path string so MLflow traces can be + salvaged. + Returns the rewritten relative path, or None if unchanged / not possible. """ path = Path(run_result_path) @@ -37,11 +45,13 @@ def rewrite_harbor_job_dir(run_result_path: Path) -> str | None: if not job.is_absolute(): return None if not job.is_dir(): - logger.warning("harbor_job_dir does not exist: %s", job) - return None + logger.warning( + "harbor_job_dir does not exist (%s); attempting relative rewrite anyway", + job, + ) run_dir = path.parent.resolve() - job_resolved = job.resolve() + job_resolved = job.resolve() if job.exists() else job for root in run_dir.parents: try: rel = job_resolved.relative_to(root) diff --git a/config/mlflow/README.md b/config/mlflow/README.md index fb41cacb..bf511c74 100644 --- a/config/mlflow/README.md +++ b/config/mlflow/README.md @@ -19,6 +19,10 @@ http://abevalflow-mlflow.ab-eval-flow.svc.cluster.local:5000 Set pipeline params `enable-mlflow=true` and `mlflow-tracking-uri` to that URL. +Client packages (`mlflow-skinny`, `pandas`) are baked into +`containers/agent-eval-harness/Containerfile`. Evaluate still pip-installs to +`/tmp` only when those imports are missing (older images). + ## Security defaults - `--allowed-hosts` lists in-cluster Service DNS + localhost only (no `*`). @@ -28,5 +32,6 @@ Set pipeline params `enable-mlflow=true` and `mlflow-tracking-uri` to that URL. ## Storage -`pvc.yaml` sets `storageClassName: gp3` (AWS). On other clusters, edit or omit -the field so the cluster default StorageClass applies. +`pvc.yaml` omits `storageClassName` so the cluster default applies. Uncomment +`storageClassName: gp3` (or your class) in that file if you need a specific +provisioner. diff --git a/config/mlflow/pvc.yaml b/config/mlflow/pvc.yaml index de313f5d..6aecd0ec 100644 --- a/config/mlflow/pvc.yaml +++ b/config/mlflow/pvc.yaml @@ -15,4 +15,6 @@ spec: resources: requests: storage: 5Gi - storageClassName: gp3 + # Omit storageClassName so the cluster default applies. Uncomment/adjust + # for a specific class (e.g. AWS EBS gp3): + # storageClassName: gp3 diff --git a/containers/agent-eval-harness/Containerfile b/containers/agent-eval-harness/Containerfile index 93ff308f..441aa1fd 100644 --- a/containers/agent-eval-harness/Containerfile +++ b/containers/agent-eval-harness/Containerfile @@ -74,13 +74,17 @@ COPY --chown=1001:0 abevalflow /opt/agent-eval-harness/abevalflow # - Anthropic SDK with Vertex AI support # - Jinja2 for template rendering # - SciPy for statistical functions +# - mlflow-skinny + pandas for post-evaluate AEH MLflow logging (evaluate.yaml +# still pip-installs to /tmp as a fallback when these imports are missing) RUN pip install --no-cache-dir \ "harbor>=0.13.0" \ "kubernetes>=32.0.0" \ pyyaml \ "anthropic[vertex]" \ jinja2 \ - scipy + scipy \ + "mlflow-skinny" \ + pandas # Set up environment ENV PYTHONPATH=/opt/agent-eval-harness \ diff --git a/pipeline/pipelines/ci-pipeline-dev.yaml b/pipeline/pipelines/ci-pipeline-dev.yaml index 35ff695d..56e051f7 100644 --- a/pipeline/pipelines/ci-pipeline-dev.yaml +++ b/pipeline/pipelines/ci-pipeline-dev.yaml @@ -187,8 +187,10 @@ spec: description: Optional prior AEH run-id for regression --baseline - name: enable-aeh-eval-check type: string - default: "true" - description: Advisory AEH harness inventory in test phase (does not fail the run) + default: "false" + description: >- + Advisory AEH harness inventory in test phase (does not fail the run). + Defaults off to avoid an extra AEH image pull; set true to opt in. # Security scanning - name: security-scan type: string diff --git a/pipeline/pipelines/ci-pipeline.yaml b/pipeline/pipelines/ci-pipeline.yaml index 48532d86..1427f47b 100644 --- a/pipeline/pipelines/ci-pipeline.yaml +++ b/pipeline/pipelines/ci-pipeline.yaml @@ -187,8 +187,10 @@ spec: description: Optional prior AEH run-id for regression --baseline - name: enable-aeh-eval-check type: string - default: "true" - description: Advisory AEH harness inventory in test phase (does not fail the run) + default: "false" + description: >- + Advisory AEH harness inventory in test phase (does not fail the run). + Defaults off to avoid an extra AEH image pull; set true to opt in. # Security scanning - name: security-scan type: string diff --git a/pipeline/tasks/phases/evaluate.yaml b/pipeline/tasks/phases/evaluate.yaml index 887af107..3a6f928a 100644 --- a/pipeline/tasks/phases/evaluate.yaml +++ b/pipeline/tasks/phases/evaluate.yaml @@ -1437,15 +1437,18 @@ spec: PARSE_AEH # Optional MLflow logging (non-blocking). - # AEH image site-packages is read-only under OpenShift SCC — install to /tmp. + # Prefer packages baked into the AEH image; if missing, install to /tmp + # (image site-packages is read-only under OpenShift SCC). if [ "$(params.enable-mlflow)" = "true" ] && [ -n "$(params.mlflow-tracking-uri)" ]; then echo "=== AEH MLflow logging ===" MLFLOW_PKG_DIR="/tmp/mlflow-pkgs" mkdir -p "$MLFLOW_PKG_DIR" export PYTHONPATH="${MLFLOW_PKG_DIR}${PYTHONPATH:+:$PYTHONPATH}" # pandas is required by AEH log_results.py (mlflow.log_table). - if ! python -c "import mlflow, pandas" 2>/dev/null; then - echo "Installing mlflow-skinny + pandas into ${MLFLOW_PKG_DIR} ..." + if python -c "import mlflow, pandas" 2>/dev/null; then + echo "mlflow/pandas already available (image or prior install); skipping pip" + else + echo "Installing mlflow-skinny + pandas into ${MLFLOW_PKG_DIR} (fallback) ..." # Do not let pip failure abort the step (set -e / pipefail). set +e python -m pip install --no-cache-dir --target "$MLFLOW_PKG_DIR" 'mlflow-skinny' 'pandas' 2>&1 | tee /tmp/mlflow-pip.log | tail -30 @@ -1722,15 +1725,17 @@ spec: PARSE_PAIRWISE # Optional MLflow logging for control + treatment (non-blocking). - # AEH image site-packages is read-only under OpenShift SCC — install to /tmp. + # Prefer packages baked into the AEH image; if missing, install to /tmp. if [ "$(params.enable-mlflow)" = "true" ] && [ -n "$(params.mlflow-tracking-uri)" ]; then echo "=== AEH MLflow logging (pairwise) ===" MLFLOW_PKG_DIR="/tmp/mlflow-pkgs" mkdir -p "$MLFLOW_PKG_DIR" export PYTHONPATH="${MLFLOW_PKG_DIR}${PYTHONPATH:+:$PYTHONPATH}" # pandas is required by AEH log_results.py (mlflow.log_table). - if ! python -c "import mlflow, pandas" 2>/dev/null; then - echo "Installing mlflow-skinny + pandas into ${MLFLOW_PKG_DIR} ..." + if python -c "import mlflow, pandas" 2>/dev/null; then + echo "mlflow/pandas already available (image or prior install); skipping pip" + else + echo "Installing mlflow-skinny + pandas into ${MLFLOW_PKG_DIR} (fallback) ..." set +e python -m pip install --no-cache-dir --target "$MLFLOW_PKG_DIR" 'mlflow-skinny' 'pandas' 2>&1 | tee /tmp/mlflow-pip.log | tail -30 PIP_RC=${PIPESTATUS[0]} diff --git a/pipeline/tasks/phases/test.yaml b/pipeline/tasks/phases/test.yaml index edc45cb1..28356f2c 100644 --- a/pipeline/tasks/phases/test.yaml +++ b/pipeline/tasks/phases/test.yaml @@ -53,10 +53,11 @@ spec: description: Submission's skip_quality_review setting from metadata.yaml - name: enable-aeh-eval-check type: string - default: "true" + default: "false" description: >- When true and eval-engine=aeh, run AEH harness inventory (advisory — - does not fail the test phase). + does not fail the test phase). Defaults off to avoid an extra AEH + image pull in the test phase; set true to opt in. - name: aeh-image type: string default: "quay.io/ecosystem-appeng/agent-eval-harness:v1.0.3" diff --git a/scripts/log_aeh_mlflow.py b/scripts/log_aeh_mlflow.py index dd8aade7..95e08d1b 100644 --- a/scripts/log_aeh_mlflow.py +++ b/scripts/log_aeh_mlflow.py @@ -1,8 +1,11 @@ """Log AEH Harbor run results to MLflow (post-evaluate, non-blocking caller). -Prefers upstream AEH ``skills/eval-mlflow/scripts/log_results.py`` when present -(under ``/opt/agent-eval-harness`` or ``AGENT_EVAL_HARNESS_ROOT``). Falls back -to a minimal metrics/params logger from ``run_result.json`` + ``summary.yaml``. +Requires an importable ``mlflow`` package in the current Python environment +(Tekton evaluate installs ``mlflow-skinny`` + ``pandas`` into ``/tmp`` when +missing). Prefers upstream AEH ``skills/eval-mlflow/scripts/log_results.py`` +when present (under ``/opt/agent-eval-harness`` or ``AGENT_EVAL_HARNESS_ROOT``). +Falls back to a minimal metrics/params logger from ``run_result.json`` + +``summary.yaml`` when upstream is absent or no-ops. Optional actions (same AEH skill tree): - ``push-feedback`` — attach judge feedback to traces (``attach_feedback.py``) @@ -37,8 +40,14 @@ logger = logging.getLogger(__name__) _AEH_MLFLOW_SCRIPTS = Path("/opt/agent-eval-harness/skills/eval-mlflow/scripts") -_AEH_LOG_RESULTS_CANDIDATES = (_AEH_MLFLOW_SCRIPTS / "log_results.py",) _DEFAULT_ACTIONS = ("log-results", "push-feedback", "sync-dataset") +# Upstream log_results.py wording varies; also catch ImportError-style lines. +_MISSING_MLFLOW_MARKERS = ( + "MLflow not installed", + "mlflow is not installed", + "No module named 'mlflow'", + 'No module named "mlflow"', +) def _aeh_script(name: str) -> Path | None: @@ -171,12 +180,22 @@ def _mlflow_importable() -> bool: return True +def _upstream_noop_missing_mlflow(combined: str, returncode: int) -> bool: + """True when upstream log_results likely skipped because mlflow is missing.""" + if any(marker in combined for marker in _MISSING_MLFLOW_MARKERS): + return True + # Structural guard: success exit but this interpreter cannot import mlflow. + if returncode == 0 and not _mlflow_importable(): + return True + return False + + def _run_aeh_log_results(*, script: Path, run_id: str, config: Path, runs_dir: Path) -> int: """Run upstream log_results.py. Returns 0 on real success. Returns non-zero when the script no-ops because - ``mlflow`` is missing (upstream exits 0 with a message — treat as failure - so we can fall back to the minimal logger). + ``mlflow`` is missing (upstream often exits 0 with a message — treat as + failure so we can fall back to the minimal logger). """ env = os.environ.copy() env["AGENT_EVAL_RUNS_DIR"] = str(runs_dir) @@ -195,7 +214,7 @@ def _run_aeh_log_results(*, script: Path, run_id: str, config: Path, runs_dir: P if result.stderr: print(result.stderr, end="", file=sys.stderr) combined = f"{result.stdout}\n{result.stderr}" - if "MLflow not installed" in combined: + if _upstream_noop_missing_mlflow(combined, result.returncode): logger.warning("AEH log_results.py skipped (mlflow package missing)") return 2 return result.returncode diff --git a/tests/test_aeh_paths.py b/tests/test_aeh_paths.py index 9d62268c..aa6a3343 100644 --- a/tests/test_aeh_paths.py +++ b/tests/test_aeh_paths.py @@ -37,3 +37,17 @@ def test_rewrite_leaves_relative_path_alone(tmp_path: Path): def test_rewrite_missing_file_returns_none(tmp_path: Path): assert rewrite_harbor_job_dir(tmp_path / "missing.json") is None + + +def test_rewrite_missing_job_dir_still_rewrites(tmp_path: Path): + """Archived/moved Harbor job dirs should still get a relative path.""" + source = tmp_path / "source" + missing_job = source / "_eval_tmp" / "aeh-jobs" / "archived-job" + run_dir = source / "reports" / "skill" / "run-1" + run_dir.mkdir(parents=True) + rr = run_dir / "run_result.json" + rr.write_text(json.dumps({"harbor_job_dir": str(missing_job.resolve())})) + + rewritten = rewrite_harbor_job_dir(rr) + assert rewritten == "_eval_tmp/aeh-jobs/archived-job" + assert json.loads(rr.read_text())["harbor_job_dir"] == rewritten diff --git a/tests/test_log_aeh_mlflow.py b/tests/test_log_aeh_mlflow.py index 881ab9a1..29d4a190 100644 --- a/tests/test_log_aeh_mlflow.py +++ b/tests/test_log_aeh_mlflow.py @@ -188,6 +188,35 @@ def test_default_schema_mapping_from_input_yaml(tmp_path: Path) -> None: assert mapping["expectations"]["reference"] == "reference.md:__file__" +def test_upstream_noop_missing_mlflow_markers() -> None: + from scripts.log_aeh_mlflow import _upstream_noop_missing_mlflow + + assert _upstream_noop_missing_mlflow("MLflow not installed\n", 0) is True + assert _upstream_noop_missing_mlflow("No module named 'mlflow'\n", 1) is True + assert _upstream_noop_missing_mlflow("Logged 3 runs\n", 0) is False + + +def test_run_aeh_log_results_detects_missing_mlflow_noop(tmp_path: Path, monkeypatch) -> None: + from scripts import log_aeh_mlflow as mod + + script = tmp_path / "log_results.py" + script.write_text("# stub\n") + + class _Result: + returncode = 0 + stdout = "MLflow not installed — skipping\n" + stderr = "" + + monkeypatch.setattr(mod.subprocess, "run", lambda *a, **k: _Result()) + rc = mod._run_aeh_log_results( + script=script, + run_id="run-1", + config=tmp_path / "eval.yaml", + runs_dir=tmp_path / "reports", + ) + assert rc == 2 + + def test_resolve_run_dir_prefers_exact_then_pairwise_prefixes(tmp_path: Path) -> None: from scripts.log_aeh_mlflow import _resolve_run_dir diff --git a/tests/test_validate_aeh.py b/tests/test_validate_aeh.py index d0ab3a63..550fa6ae 100644 --- a/tests/test_validate_aeh.py +++ b/tests/test_validate_aeh.py @@ -290,6 +290,44 @@ def test_single_mode_ignores_pairwise_configs(self, tmp_path): assert errors == [] +class TestAEHPluginDirsValidation: + """runner.plugin_dirs must exist and contain SKILL.md.""" + + def test_missing_plugin_dir(self, tmp_path): + eval_yaml = { + **VALID_EVAL_YAML, + "skill": "my-aeh-eval", + "runner": {"plugin_dirs": ["skills"]}, + } + create_aeh_submission(tmp_path, eval_yaml_content=eval_yaml) + errors = validate_submission(tmp_path, eval_engine=EvalEngine.AEH) + assert any("plugin_dirs entry 'skills/' is missing" in e for e in errors) + + def test_plugin_dir_without_skill_md(self, tmp_path): + eval_yaml = { + **VALID_EVAL_YAML, + "skill": "my-aeh-eval", + "runner": {"plugin_dirs": ["skills"]}, + } + create_aeh_submission(tmp_path, eval_yaml_content=eval_yaml) + (tmp_path / "skills").mkdir() + errors = validate_submission(tmp_path, eval_engine=EvalEngine.AEH) + assert any("must contain SKILL.md" in e for e in errors) + + def test_nested_skill_md_passes(self, tmp_path): + eval_yaml = { + **VALID_EVAL_YAML, + "skill": "my-aeh-eval", + "runner": {"plugin_dirs": ["skills"]}, + } + create_aeh_submission(tmp_path, eval_yaml_content=eval_yaml) + skill_dir = tmp_path / "skills" / "my-aeh-eval" + skill_dir.mkdir(parents=True) + (skill_dir / "SKILL.md").write_text("# skill\n") + errors = validate_submission(tmp_path, eval_engine=EvalEngine.AEH) + assert errors == [] + + class TestAEHToolInterceptionValidation: """inputs.tools / permissions / tool_handlers.yaml checks.""" From 118d14d975af8233ac1597345bbdda730c9bcb41 Mon Sep 17 00:00:00 2001 From: gziv Date: Sun, 2 Aug 2026 12:12:50 +0300 Subject: [PATCH 09/12] fix(aeh): drop structural mlflow noop guard that breaks CI Rely on upstream message markers only so exit-0 success output is not misclassified when mlflow is absent from the test environment. --- scripts/log_aeh_mlflow.py | 19 ++++++++++--------- tests/test_log_aeh_mlflow.py | 6 +++--- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/scripts/log_aeh_mlflow.py b/scripts/log_aeh_mlflow.py index 95e08d1b..c24388aa 100644 --- a/scripts/log_aeh_mlflow.py +++ b/scripts/log_aeh_mlflow.py @@ -180,14 +180,15 @@ def _mlflow_importable() -> bool: return True -def _upstream_noop_missing_mlflow(combined: str, returncode: int) -> bool: - """True when upstream log_results likely skipped because mlflow is missing.""" - if any(marker in combined for marker in _MISSING_MLFLOW_MARKERS): - return True - # Structural guard: success exit but this interpreter cannot import mlflow. - if returncode == 0 and not _mlflow_importable(): - return True - return False +def _upstream_noop_missing_mlflow(combined: str) -> bool: + """True when upstream log_results output indicates mlflow was missing. + + Relies on known upstream / ImportError message markers only. A structural + ``returncode == 0 and not importable`` guard is intentionally avoided: it + false-positives in test envs (and any interpreter) where mlflow is absent + but output is a real success string such as ``Logged N runs``. + """ + return any(marker in combined for marker in _MISSING_MLFLOW_MARKERS) def _run_aeh_log_results(*, script: Path, run_id: str, config: Path, runs_dir: Path) -> int: @@ -214,7 +215,7 @@ def _run_aeh_log_results(*, script: Path, run_id: str, config: Path, runs_dir: P if result.stderr: print(result.stderr, end="", file=sys.stderr) combined = f"{result.stdout}\n{result.stderr}" - if _upstream_noop_missing_mlflow(combined, result.returncode): + if _upstream_noop_missing_mlflow(combined): logger.warning("AEH log_results.py skipped (mlflow package missing)") return 2 return result.returncode diff --git a/tests/test_log_aeh_mlflow.py b/tests/test_log_aeh_mlflow.py index 29d4a190..2bcfaf0e 100644 --- a/tests/test_log_aeh_mlflow.py +++ b/tests/test_log_aeh_mlflow.py @@ -191,9 +191,9 @@ def test_default_schema_mapping_from_input_yaml(tmp_path: Path) -> None: def test_upstream_noop_missing_mlflow_markers() -> None: from scripts.log_aeh_mlflow import _upstream_noop_missing_mlflow - assert _upstream_noop_missing_mlflow("MLflow not installed\n", 0) is True - assert _upstream_noop_missing_mlflow("No module named 'mlflow'\n", 1) is True - assert _upstream_noop_missing_mlflow("Logged 3 runs\n", 0) is False + assert _upstream_noop_missing_mlflow("MLflow not installed\n") is True + assert _upstream_noop_missing_mlflow("No module named 'mlflow'\n") is True + assert _upstream_noop_missing_mlflow("Logged 3 runs\n") is False def test_run_aeh_log_results_detects_missing_mlflow_noop(tmp_path: Path, monkeypatch) -> None: From 5ec043410de4bef7ab32d21280649eb81b7ac233 Mon Sep 17 00:00:00 2001 From: gziv Date: Sun, 2 Aug 2026 14:36:29 +0300 Subject: [PATCH 10/12] fix(aeh): one MLflow experiment per PipelineRun Force --experiment to the PipelineRun id so control/treatment share one experiment as sibling runs; patch AEH config skill to the reports folder. --- pipeline/tasks/phases/evaluate.yaml | 10 + scripts/log_aeh_mlflow.py | 297 +++++++++++++++++++++++----- tests/test_log_aeh_mlflow.py | 99 ++++++++++ 3 files changed, 352 insertions(+), 54 deletions(-) diff --git a/pipeline/tasks/phases/evaluate.yaml b/pipeline/tasks/phases/evaluate.yaml index 3a6f928a..5e8b6797 100644 --- a/pipeline/tasks/phases/evaluate.yaml +++ b/pipeline/tasks/phases/evaluate.yaml @@ -1461,9 +1461,11 @@ spec: fi if python -c "import mlflow, pandas; print('mlflow OK', mlflow.__version__, 'pandas', pandas.__version__)"; then export MLFLOW_TRACKING_URI="$(params.mlflow-tracking-uri)" + # One MLflow experiment per PipelineRun; this AEH run is one MLflow run. python "$PIPELINE_DIR/scripts/log_aeh_mlflow.py" \ --enabled true \ --tracking-uri "$(params.mlflow-tracking-uri)" \ + --experiment "$RUN_ID" \ --run-id "$RUN_ID" \ --config "$SUBMISSION_DIR/eval.yaml" \ --runs-dir "$(workspaces.source.path)/reports" \ @@ -1747,17 +1749,25 @@ spec: fi if python -c "import mlflow, pandas; print('mlflow OK', mlflow.__version__, 'pandas', pandas.__version__)"; then export MLFLOW_TRACKING_URI="$(params.mlflow-tracking-uri)" + # One MLflow experiment per PipelineRun; control/treatment are runs under it. for VARIANT_RUN in "control-$RUN_ID" "treatment-$RUN_ID"; do CFG="$SUBMISSION_DIR/$TREATMENT_CONFIG" if [[ "$VARIANT_RUN" == control-* ]]; then CFG="$SUBMISSION_DIR/$CONTROL_CONFIG" fi + # sync-dataset once (first variant); later variants only log + feedback. + ACTIONS="log-results,push-feedback,sync-dataset" + if [[ "$VARIANT_RUN" != control-* ]]; then + ACTIONS="log-results,push-feedback" + fi python "$PIPELINE_DIR/scripts/log_aeh_mlflow.py" \ --enabled true \ --tracking-uri "$(params.mlflow-tracking-uri)" \ + --experiment "$RUN_ID" \ --run-id "$VARIANT_RUN" \ --config "$CFG" \ --runs-dir "$REPORTS_DIR" \ + --actions "$ACTIONS" \ 2>&1 || echo "Warning: MLflow logging failed for $VARIANT_RUN (non-blocking)" done else diff --git a/scripts/log_aeh_mlflow.py b/scripts/log_aeh_mlflow.py index c24388aa..4ca186ae 100644 --- a/scripts/log_aeh_mlflow.py +++ b/scripts/log_aeh_mlflow.py @@ -12,6 +12,17 @@ - ``sync-dataset`` — sync cases to the MLflow dataset registry when a schema mapping is available or a simple ``input.yaml:prompt`` default applies +Experiment naming +----------------- +Pass ``--experiment `` so **one PipelineRun maps to one +MLflow experiment**. Control/treatment (or N attempts) are separate MLflow +*runs* under that experiment (``--run-id``), not separate experiments. + +When ``--experiment`` is set we temporarily patch the eval config so AEH +scripts use that experiment name and the reports skill directory that +actually contains ``--run-id`` (pairwise control yaml skill names often +differ from the shared reports folder). + Usage:: python scripts/log_aeh_mlflow.py \\ @@ -19,6 +30,7 @@ --config /path/to/eval.yaml \\ --runs-dir /workspace/source/reports \\ --tracking-uri http://mlflow.example:5000 \\ + --experiment \\ [--actions log-results,push-feedback,sync-dataset] Skip (exit 0) when ``--enabled false`` or tracking URI is empty. @@ -71,6 +83,105 @@ def _parse_actions(raw: str) -> list[str]: return actions or list(_DEFAULT_ACTIONS) +def _variant_from_run_id(run_id: str) -> str | None: + if run_id.startswith("control-"): + return "control" + if run_id.startswith("treatment-"): + return "treatment" + return None + + +def _discover_run_dir(runs_dir: Path, run_id: str) -> Path | None: + """Find ``runs_dir//`` by scanning skill directories.""" + if not runs_dir.is_dir(): + return None + exact_hits: list[Path] = [] + for skill_dir in sorted(p for p in runs_dir.iterdir() if p.is_dir()): + candidate = skill_dir / run_id + if candidate.is_dir(): + exact_hits.append(candidate) + if len(exact_hits) == 1: + return exact_hits[0] + if len(exact_hits) > 1: + logger.warning( + "Multiple report dirs for run_id=%r; using first: %s", + run_id, + exact_hits[0], + ) + return exact_hits[0] + return None + + +def _resolve_run_dir(runs_dir: Path, skill: str, run_id: str) -> Path: + """Locate report dir for ``run_id``, tolerating skill / pairwise mismatches. + + Order: + 1. ``runs_dir//`` + 2. Pairwise prefixes under ``skill`` when ``run_id`` is bare + 3. Scan all skill dirs under ``runs_dir`` for ```` + """ + candidates = [runs_dir / skill / run_id] + if not run_id.startswith(("control-", "treatment-")): + candidates.append(runs_dir / skill / f"treatment-{run_id}") + candidates.append(runs_dir / skill / f"control-{run_id}") + for candidate in candidates: + if candidate.is_dir(): + return candidate + + discovered = _discover_run_dir(runs_dir, run_id) + if discovered is not None: + return discovered + + tried = ", ".join(str(c) for c in candidates) + raise FileNotFoundError(f"AEH run dir not found for skill={skill!r} run_id={run_id!r}; tried: {tried}") + + +def _reports_skill_name(runs_dir: Path, config_skill: str, run_id: str) -> str: + """Skill folder name under reports/ that contains ``run_id``.""" + try: + run_dir = _resolve_run_dir(runs_dir, config_skill, run_id) + except FileNotFoundError: + return config_skill + return run_dir.parent.name + + +def _write_patched_config( + config: Path, + *, + experiment: str, + reports_skill: str, +) -> Path: + """Write a temp eval.yaml forcing AEH experiment + reports skill path.""" + raw = yaml.safe_load(config.read_text()) or {} + if not isinstance(raw, dict): + raw = {} + raw["skill"] = reports_skill + # AEH EvalConfig.name / eval_name fall back to skill; keep name aligned. + if "name" in raw: + raw["name"] = reports_skill + mlflow_cfg = raw.get("mlflow") if isinstance(raw.get("mlflow"), dict) else {} + mlflow_cfg = dict(mlflow_cfg) + mlflow_cfg["experiment"] = experiment + raw["mlflow"] = mlflow_cfg + + tmp = tempfile.NamedTemporaryFile( + mode="w", + suffix=".yaml", + delete=False, + prefix="aeh-mlflow-config-", + ) + yaml.safe_dump(raw, tmp, sort_keys=False, allow_unicode=True) + tmp.close() + path = Path(tmp.name) + logger.info( + "Patched AEH MLflow config: experiment=%s skill=%s -> %s", + experiment, + reports_skill, + path, + ) + return path + + def _default_schema_mapping(config: Path) -> dict | None: """Build a minimal mapping when cases expose ``input.yaml`` with ``prompt``.""" try: @@ -121,12 +232,15 @@ def _maybe_sync_dataset(*, config: Path, runs_dir: Path) -> None: return mapping_path = config.parent / "schema_mapping.json" + # Patched temp configs live outside the submission; map from original parent + # is handled by caller passing the original config's mapping via sibling. tmp_path: Path | None = None if mapping_path.is_file(): logger.info("Using schema mapping: %s", mapping_path) else: mapping = _default_schema_mapping(config) if mapping is None: + # When config is a temp patch, try schema next to nothing — skip. logger.info("No schema_mapping.json and no default input.yaml:prompt mapping; skipping sync-dataset") return tmp = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False, prefix="aeh-schema-mapping-") @@ -221,7 +335,14 @@ def _run_aeh_log_results(*, script: Path, run_id: str, config: Path, runs_dir: P return result.returncode -def _do_log_results(*, run_id: str, config: Path, runs_dir: Path, tracking_uri: str) -> int: +def _do_log_results( + *, + run_id: str, + config: Path, + runs_dir: Path, + tracking_uri: str, + experiment: str | None, +) -> int: """Log results via AEH script or minimal fallback. Returns 0 on success.""" script = _resolve_log_results_script() if script is not None: @@ -241,6 +362,7 @@ def _do_log_results(*, run_id: str, config: Path, runs_dir: Path, tracking_uri: config=config, runs_dir=runs_dir, tracking_uri=tracking_uri, + experiment=experiment, ) except Exception: logger.exception("MLflow logging failed") @@ -248,24 +370,14 @@ def _do_log_results(*, run_id: str, config: Path, runs_dir: Path, tracking_uri: return 0 -def _resolve_run_dir(runs_dir: Path, skill: str, run_id: str) -> Path: - """Locate ``runs_dir//``, with pairwise prefix fallbacks. - - Pairwise AEH layouts use ``control-`` / ``treatment-``. When callers - pass a bare pipeline-run id, prefer the treatment dir, then control. - """ - candidates = [runs_dir / skill / run_id] - if not run_id.startswith(("control-", "treatment-")): - candidates.append(runs_dir / skill / f"treatment-{run_id}") - candidates.append(runs_dir / skill / f"control-{run_id}") - for candidate in candidates: - if candidate.is_dir(): - return candidate - tried = ", ".join(str(c) for c in candidates) - raise FileNotFoundError(f"AEH run dir not found for skill={skill!r} run_id={run_id!r}; tried: {tried}") - - -def _minimal_mlflow_log(*, run_id: str, config: Path, runs_dir: Path, tracking_uri: str) -> None: +def _minimal_mlflow_log( + *, + run_id: str, + config: Path, + runs_dir: Path, + tracking_uri: str, + experiment: str | None = None, +) -> None: try: import mlflow except ImportError as exc: @@ -274,7 +386,7 @@ def _minimal_mlflow_log(*, run_id: str, config: Path, runs_dir: Path, tracking_u raw = yaml.safe_load(config.read_text()) or {} skill = str(raw.get("skill") or config.parent.name) mlflow_cfg = raw.get("mlflow") if isinstance(raw.get("mlflow"), dict) else {} - experiment = str(mlflow_cfg.get("experiment") or skill or "abevalflow-aeh") + exp_name = (experiment or "").strip() or str(mlflow_cfg.get("experiment") or skill or "abevalflow-aeh") run_dir = _resolve_run_dir(runs_dir, skill, run_id) @@ -291,11 +403,15 @@ def _minimal_mlflow_log(*, run_id: str, config: Path, runs_dir: Path, tracking_u summary = loaded mlflow.set_tracking_uri(tracking_uri) - mlflow.set_experiment(experiment) + mlflow.set_experiment(exp_name) with mlflow.start_run(run_name=run_id): mlflow.log_param("skill", skill) mlflow.log_param("run_id", run_id) + mlflow.log_param("pipeline_experiment", exp_name) mlflow.log_param("execution_mode", run_result.get("execution_mode", "harbor")) + variant = _variant_from_run_id(run_id) + if variant: + mlflow.log_param("variant", variant) if run_result.get("model"): mlflow.log_param("model", run_result["model"]) @@ -326,7 +442,7 @@ def _minimal_mlflow_log(*, run_id: str, config: Path, runs_dir: Path, tracking_u # artifact root (mlflow-artifacts:/) on the tracking server. logger.warning("MLflow artifact upload failed for %s", artifact, exc_info=True) - logger.info("Minimal MLflow log complete: experiment=%s run_id=%s", experiment, run_id) + logger.info("Minimal MLflow log complete: experiment=%s run_id=%s", exp_name, run_id) def main(argv: list[str] | None = None) -> int: @@ -336,6 +452,15 @@ def main(argv: list[str] | None = None) -> int: parser.add_argument("--config", type=Path, required=True) parser.add_argument("--runs-dir", type=Path, required=True) parser.add_argument("--tracking-uri", default=os.environ.get("MLFLOW_TRACKING_URI", "")) + parser.add_argument( + "--experiment", + default="", + help=( + "MLflow experiment name for this PipelineRun. When set, all variants " + "(control/treatment) share this one experiment. Prefer the Tekton " + "PipelineRun name." + ), + ) parser.add_argument( "--enabled", default="true", @@ -375,44 +500,108 @@ def main(argv: list[str] | None = None) -> int: actions = _parse_actions(args.actions) logger.info("MLflow actions: %s", ", ".join(actions)) - # Belt-and-suspenders: AEH log_results rejects absolute harbor_job_dir. - try: - from abevalflow.harbor_extensions.aeh_paths import rewrite_harbor_job_dir - - raw_cfg = yaml.safe_load(config.read_text()) or {} - skill = str(raw_cfg.get("skill") or config.parent.name) - rr = runs_dir / skill / args.run_id / "run_result.json" - if rr.is_file(): - rewritten = rewrite_harbor_job_dir(rr) - if rewritten: - logger.info("Rewrote harbor_job_dir -> %s", rewritten) - except Exception: - logger.warning("harbor_job_dir rewrite failed; continuing", exc_info=True) + experiment = (args.experiment or "").strip() + raw_cfg = yaml.safe_load(config.read_text()) or {} + config_skill = str(raw_cfg.get("skill") or config.parent.name) if isinstance(raw_cfg, dict) else config.parent.name + reports_skill = _reports_skill_name(runs_dir, config_skill, args.run_id) + + patched_config: Path | None = None + effective_config = config + if experiment: + patched_config = _write_patched_config( + config, + experiment=experiment, + reports_skill=reports_skill, + ) + effective_config = patched_config + logger.info( + "One experiment per PipelineRun: experiment=%s reports_skill=%s run_id=%s", + experiment, + reports_skill, + args.run_id, + ) - # sync-dataset is run-independent; do it first so the registry is warm. - if "sync-dataset" in actions: + try: + # Belt-and-suspenders: AEH log_results rejects absolute harbor_job_dir. try: - _maybe_sync_dataset(config=config, runs_dir=runs_dir) + from abevalflow.harbor_extensions.aeh_paths import rewrite_harbor_job_dir + + try: + run_dir = _resolve_run_dir(runs_dir, reports_skill, args.run_id) + except FileNotFoundError: + run_dir = None + if run_dir is not None: + rr = run_dir / "run_result.json" + if rr.is_file(): + rewritten = rewrite_harbor_job_dir(rr) + if rewritten: + logger.info("Rewrote harbor_job_dir -> %s", rewritten) except Exception: - logger.warning("sync-dataset failed (non-blocking)", exc_info=True) + logger.warning("harbor_job_dir rewrite failed; continuing", exc_info=True) - log_rc = 0 - if "log-results" in actions: - log_rc = _do_log_results( - run_id=args.run_id, - config=config, - runs_dir=runs_dir, - tracking_uri=tracking_uri, - ) + # sync-dataset resolves dataset.path relative to config.parent — keep a + # patch beside the submission when overriding experiment. + if "sync-dataset" in actions: + sync_config: Path | None = None + try: + if experiment: + sync_config = _write_submission_local_patch( + config, + experiment=experiment, + reports_skill=reports_skill, + ) + else: + sync_config = config + _maybe_sync_dataset(config=sync_config, runs_dir=runs_dir) + except Exception: + logger.warning("sync-dataset failed (non-blocking)", exc_info=True) + finally: + if sync_config is not None and sync_config != config: + sync_config.unlink(missing_ok=True) + + log_rc = 0 + if "log-results" in actions: + log_rc = _do_log_results( + run_id=args.run_id, + config=effective_config, + runs_dir=runs_dir, + tracking_uri=tracking_uri, + experiment=experiment or None, + ) + + if "push-feedback" in actions: + try: + _maybe_push_feedback(run_id=args.run_id, config=effective_config, runs_dir=runs_dir) + except Exception: + logger.warning("push-feedback failed (non-blocking)", exc_info=True) + + # Keep historical behavior: non-zero only when log-results itself fails. + return log_rc + finally: + if patched_config is not None: + patched_config.unlink(missing_ok=True) - if "push-feedback" in actions: - try: - _maybe_push_feedback(run_id=args.run_id, config=config, runs_dir=runs_dir) - except Exception: - logger.warning("push-feedback failed (non-blocking)", exc_info=True) - # Keep historical behavior: non-zero only when log-results itself fails. - return log_rc +def _write_submission_local_patch( + config: Path, + *, + experiment: str, + reports_skill: str, +) -> Path: + """Patch next to the submission so dataset.path stays resolvable.""" + raw = yaml.safe_load(config.read_text()) or {} + if not isinstance(raw, dict): + raw = {} + raw["skill"] = reports_skill + if "name" in raw: + raw["name"] = reports_skill + mlflow_cfg = raw.get("mlflow") if isinstance(raw.get("mlflow"), dict) else {} + mlflow_cfg = dict(mlflow_cfg) + mlflow_cfg["experiment"] = experiment + raw["mlflow"] = mlflow_cfg + path = config.parent / f".abevalflow-mlflow-{os.getpid()}.yaml" + path.write_text(yaml.safe_dump(raw, sort_keys=False, allow_unicode=True)) + return path if __name__ == "__main__": diff --git a/tests/test_log_aeh_mlflow.py b/tests/test_log_aeh_mlflow.py index 2bcfaf0e..aa124902 100644 --- a/tests/test_log_aeh_mlflow.py +++ b/tests/test_log_aeh_mlflow.py @@ -173,6 +173,105 @@ def log_artifact(self, path): assert ("tokens_input", 100.0) in calls["metrics"] +def test_minimal_logger_uses_pipeline_experiment_override(tmp_path: Path, monkeypatch) -> None: + """--experiment forces one experiment per PipelineRun (ignores eval.yaml mlflow.experiment).""" + runs, config = _write_run(tmp_path) + calls: dict[str, list] = {"params": []} + + class _FakeRun: + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + class _FakeMlflow: + def set_tracking_uri(self, uri): + pass + + def set_experiment(self, name): + calls["experiment"] = name + + def start_run(self, run_name=None): + calls["run_name"] = run_name + return _FakeRun() + + def log_param(self, k, v): + calls["params"].append((k, v)) + + def log_metric(self, k, v): + pass + + def log_artifact(self, path): + pass + + import sys + + monkeypatch.setitem(sys.modules, "mlflow", _FakeMlflow()) + monkeypatch.setattr("scripts.log_aeh_mlflow._resolve_log_results_script", lambda: None) + rc = main( + [ + "--run-id", + "run-1", + "--config", + str(config), + "--runs-dir", + str(runs), + "--tracking-uri", + "http://mlflow.example:5000", + "--experiment", + "aeh-mlflow-single-abc123", + "--enabled", + "true", + "--actions", + "log-results", + ] + ) + assert rc == 0 + assert calls["experiment"] == "aeh-mlflow-single-abc123" + assert calls["run_name"] == "run-1" + assert ("pipeline_experiment", "aeh-mlflow-single-abc123") in calls["params"] + + +def test_resolve_run_dir_discovers_when_config_skill_mismatches(tmp_path: Path) -> None: + """Pairwise control yaml skill often differs from the shared reports folder.""" + from scripts.log_aeh_mlflow import _resolve_run_dir, _reports_skill_name, _write_patched_config + + runs = tmp_path / "reports" + run_id = "control-pr-1" + real_skill = "aeh-hello-world-pairwise" + (runs / real_skill / run_id).mkdir(parents=True) + config_skill = "aeh-hello-world-pairwise-control" + found = _resolve_run_dir(runs, config_skill, run_id) + assert found == runs / real_skill / run_id + assert _reports_skill_name(runs, config_skill, run_id) == real_skill + + cfg = tmp_path / "eval-control.yaml" + cfg.write_text( + yaml.safe_dump( + { + "skill": config_skill, + "mlflow": {"experiment": "aeh-hello-world-pairwise-control"}, + } + ) + ) + patched = _write_patched_config(cfg, experiment="pr-1", reports_skill=real_skill) + try: + data = yaml.safe_load(patched.read_text()) + assert data["skill"] == real_skill + assert data["mlflow"]["experiment"] == "pr-1" + finally: + patched.unlink(missing_ok=True) + + +def test_variant_from_run_id() -> None: + from scripts.log_aeh_mlflow import _variant_from_run_id + + assert _variant_from_run_id("control-foo") == "control" + assert _variant_from_run_id("treatment-foo") == "treatment" + assert _variant_from_run_id("foo") is None + + def test_default_schema_mapping_from_input_yaml(tmp_path: Path) -> None: from scripts.log_aeh_mlflow import _default_schema_mapping From 1d42effc8d3b540cc6f43579112a48c350f74111 Mon Sep 17 00:00:00 2001 From: gziv Date: Sun, 2 Aug 2026 14:43:40 +0300 Subject: [PATCH 11/12] fix(test): sort imports in test_log_aeh_mlflow for ruff I001 --- tests/test_log_aeh_mlflow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_log_aeh_mlflow.py b/tests/test_log_aeh_mlflow.py index aa124902..78110098 100644 --- a/tests/test_log_aeh_mlflow.py +++ b/tests/test_log_aeh_mlflow.py @@ -235,7 +235,7 @@ def log_artifact(self, path): def test_resolve_run_dir_discovers_when_config_skill_mismatches(tmp_path: Path) -> None: """Pairwise control yaml skill often differs from the shared reports folder.""" - from scripts.log_aeh_mlflow import _resolve_run_dir, _reports_skill_name, _write_patched_config + from scripts.log_aeh_mlflow import _reports_skill_name, _resolve_run_dir, _write_patched_config runs = tmp_path / "reports" run_id = "control-pr-1" From d24c5f8af9e13007a109e3e697bcfc627357ee0e Mon Sep 17 00:00:00 2001 From: gziv Date: Sun, 2 Aug 2026 15:21:03 +0300 Subject: [PATCH 12/12] chore(aeh): bake full mlflow[genai] into v1.0.4 runner image Pin notes to AEH 1.30.1; prefer full mlflow over skinny so Harbor trace export APIs are available in the evaluate image. --- containers/agent-eval-harness/Containerfile | 20 +++++++++++--------- 1 file changed, 11 insertions(+), 9 deletions(-) diff --git a/containers/agent-eval-harness/Containerfile b/containers/agent-eval-harness/Containerfile index 441aa1fd..f6aa20b7 100644 --- a/containers/agent-eval-harness/Containerfile +++ b/containers/agent-eval-harness/Containerfile @@ -5,17 +5,17 @@ # in a Kubernetes/OpenShift environment. # # Build (from repo root — COPY abevalflow requires repo-root context): -# podman build -t quay.io/ecosystem-appeng/agent-eval-harness:v1.0.3 \ +# podman build -t quay.io/ecosystem-appeng/agent-eval-harness:v1.0.4 \ # --build-arg AEH_SHA= \ # -f containers/agent-eval-harness/Containerfile . # # Required build args (PIN BEFORE BUILDING): # AEH_SHA: Commit SHA of agent-eval-harness repo (7-40 hex chars; no branch/HEAD) # -# Published smoke-tested image: quay.io/ecosystem-appeng/agent-eval-harness:v1.0.3 -# Rebuilds from this Containerfile must pin AEH_SHA to the same upstream commit -# used for that tag (inspect image labels / build logs) — do not leave the -# REPLACE_WITH_COMMIT_SHA placeholder. +# Published smoke-tested image: quay.io/ecosystem-appeng/agent-eval-harness:v1.0.4 +# (AEH upstream 1.30.1 / 75f7188299f6594425175ef71ec41a7a6a6f6ffd — rebuild with +# that SHA or a newer main tip). Do not leave REPLACE_WITH_COMMIT_SHA. + # # Note: Uses official Harbor from PyPI (>=0.13.0), not the skills_eval_corrections fork. # The fork's OpenShift backend is only needed for Harbor engine evaluations, not AEH. @@ -32,7 +32,8 @@ LABEL name="agent-eval-harness" \ vendor="Red Hat Ecosystem App Engineering" \ description="Agent-Eval-Harness runner for ABEvalFlow pipeline" \ io.k8s.display-name="AEH Runner" \ - io.openshift.tags="abevalflow,evaluation,agent-eval-harness" + io.openshift.tags="abevalflow,evaluation,agent-eval-harness" \ + abevalflow.aeh.sha="${AEH_SHA}" USER 0 @@ -74,8 +75,9 @@ COPY --chown=1001:0 abevalflow /opt/agent-eval-harness/abevalflow # - Anthropic SDK with Vertex AI support # - Jinja2 for template rendering # - SciPy for statistical functions -# - mlflow-skinny + pandas for post-evaluate AEH MLflow logging (evaluate.yaml -# still pip-installs to /tmp as a fallback when these imports are missing) +# - mlflow[genai] + pandas for post-evaluate AEH MLflow logging/traces +# (full mlflow — skinny breaks async trace export / AsyncExportQueue). +# evaluate.yaml still pip-installs to /tmp as a fallback for older images. RUN pip install --no-cache-dir \ "harbor>=0.13.0" \ "kubernetes>=32.0.0" \ @@ -83,7 +85,7 @@ RUN pip install --no-cache-dir \ "anthropic[vertex]" \ jinja2 \ scipy \ - "mlflow-skinny" \ + "mlflow[genai]" \ pandas # Set up environment