diff --git a/eval/rhdh-templates/uplift/run-local.sh b/eval/rhdh-templates/uplift/run-local.sh index e5a88e8..6755e3f 100755 --- a/eval/rhdh-templates/uplift/run-local.sh +++ b/eval/rhdh-templates/uplift/run-local.sh @@ -24,18 +24,19 @@ fi cd "${REPO_ROOT}" export UV_CACHE_DIR="${UV_CACHE_DIR:-/tmp/rhdh-aeh-uv-cache}" export AGENT_EVAL_RUNS_DIR="${REPO_ROOT}/eval/runs/rhdh-templates-uplift" +prefix="uplift-$(date -u +%Y%m%dT%H%M%SZ)" for arm in skill baseline; do config="${SCRIPT_DIR}/${arm}.eval.yaml" for index in $(seq 1 "${runs}"); do - run_id="uplift-${arm}-${index}" + run_id="${prefix}-${arm}-${index}" uv run --project "${aeh_checkout}" python "${SCRIPT_DIR}/run.py" --aeh-dir "${aeh_checkout}" --config "${config}" --model "${MODEL}" --run-id "${run_id}" done done -comparison_dir="/tmp/rhdh-templates-uplift-comparison" +comparison_dir="/tmp/rhdh-templates-uplift-comparison-${prefix}" comparison_input="$(mktemp -d /tmp/rhdh-templates-uplift-runs.XXXXXX)" for index in $(seq 1 "${runs}"); do - cp -a "${AGENT_EVAL_RUNS_DIR}/rhdh-templates/uplift-skill-${index}" "${comparison_input}/" - cp -a "${AGENT_EVAL_RUNS_DIR}/rhdh-templates-uplift-baseline/uplift-baseline-${index}" "${comparison_input}/" + cp -a "${AGENT_EVAL_RUNS_DIR}/rhdh-templates/${prefix}-skill-${index}" "${comparison_input}/" + cp -a "${AGENT_EVAL_RUNS_DIR}/rhdh-templates-uplift-baseline/${prefix}-baseline-${index}" "${comparison_input}/" done uv run --project "${aeh_checkout}" python "${aeh_checkout}/skills/eval-compare/scripts/compare.py" generate "${comparison_input}" --output "${comparison_dir}" --title "rhdh-templates uplift" --overview "Observed comparison only; no generalization beyond this reviewed case set." echo "Comparison: ${comparison_dir}" diff --git a/tests/unit/test_rhdh_templates_nunjucks_uplift.py b/tests/unit/test_rhdh_templates_nunjucks_uplift.py index c53e5f0..e53b98f 100644 --- a/tests/unit/test_rhdh_templates_nunjucks_uplift.py +++ b/tests/unit/test_rhdh_templates_nunjucks_uplift.py @@ -77,5 +77,9 @@ def test_nunjucks_uplift_runs_three_local_attempts_per_arm_with_case_comparison( assert "runs=3" in command assert "for arm in skill baseline; do" in command assert 'for index in $(seq 1 "${runs}"); do' in command + assert 'prefix="uplift-$(date -u +%Y%m%dT%H%M%SZ)"' in command + assert 'run_id="${prefix}-${arm}-${index}"' in command + assert "${prefix}-skill-${index}" in command + assert "${prefix}-baseline-${index}" in command assert "compare.py" in command assert "Observed comparison only" in command