# GitLab CI component template for decentralized enterprise evals.
#
# Copy into a CI component project or include from product repos after adapting
# the image and runner command to local infrastructure.

spec:
  inputs:
    stage:
      default: test
    runner_image:
      default: registry.example.com/ai-platform/eval-runner:1.0.0
    pack_glob:
      default: evals/packs/*/manifest.yaml
    output_dir:
      default: reports/evals
    team_id:
      default: $CI_PROJECT_NAMESPACE
    runner_command:
      default: ./evals/runners/run_eval.sh
    fail_on_release_blocking:
      default: "true"
    publish_annotations:
      default: "true"
---

enterprise_ai_eval:
  stage: $[[ inputs.stage ]]
  image: $[[ inputs.runner_image ]]
  variables:
    PACK_GLOB: $[[ inputs.pack_glob ]]
    OUTPUT_DIR: $[[ inputs.output_dir ]]
    TEAM_ID: $[[ inputs.team_id ]]
    FAIL_ON_RELEASE_BLOCKING: $[[ inputs.fail_on_release_blocking ]]
    PUBLISH_ANNOTATIONS: $[[ inputs.publish_annotations ]]
  script:
    - mkdir -p "$OUTPUT_DIR"
    - $[[ inputs.runner_command ]] --pack-glob "$PACK_GLOB" --output-dir "$OUTPUT_DIR"
    - scripts/.venv-tts/bin/python3 -m statebench.runners.enterprise_eval_contract_smoke
    - test -f "$OUTPUT_DIR/eval-results.jsonl"
    - scripts/.venv-tts/bin/python3 -m statebench.runners.contract_validator --schema eval_result_v1 --input "$OUTPUT_DIR/eval-results.jsonl" --jsonl
    - scripts/.venv-tts/bin/python3 -m statebench.runners.eval_rollup_collector --eval-results "$OUTPUT_DIR/eval-results.jsonl" --team-id "$TEAM_ID" --team-rollup-id "team_${CI_PROJECT_ID}_${CI_PIPELINE_ID}" --firm-rollup-id "firm_${CI_PIPELINE_ID}" --window-start-utc "${CI_PIPELINE_CREATED_AT}" --window-end-utc "${CI_JOB_STARTED_AT}" --created-at-utc "${CI_JOB_STARTED_AT}" --team-output "$OUTPUT_DIR/team-rollup.json" --firm-output "$OUTPUT_DIR/firm-rollup.json" --team-notes-ref "$OUTPUT_DIR/team-summary.md"
    - test -f "$OUTPUT_DIR/annotations.json"
    - scripts/.venv-tts/bin/python3 -m statebench.runners.contract_validator --schema gitlab_eval_annotation_v1 --input "$OUTPUT_DIR/annotations.json"
  artifacts:
    when: always
    expire_in: 30 days
    paths:
      - $[[ inputs.output_dir ]]/
    reports:
      junit: $[[ inputs.output_dir ]]/junit.xml
      annotations: $[[ inputs.output_dir ]]/annotations.json
  rules:
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    - if: $CI_PIPELINE_SOURCE == "schedule"