{"schema_version":"native_eval_fixture_v1","fixture_id":"native_promptfoo_evaluate_result_source_observed_001","framework":"Promptfoo","framework_version":"source-main-2026-06-22","fixture_type":"source_observed_contract","native_grain":"per-result EvaluateResult row","primary_source_refs":["https://github.com/promptfoo/promptfoo/blob/main/src/types/index.ts","https://github.com/promptfoo/promptfoo/blob/main/src/util/output.ts","https://www.promptfoo.dev/docs/usage/command-line/"],"source_retrieved_at":"2026-06-22","payload":{"id":"eval-result-001","description":"HR policy citation case","promptIdx":0,"testIdx":0,"testCase":{"vars":{"question":"Can a US employee carry over unused PTO into next year?","persona":"hr_business_partner"},"assert":[{"type":"contains-json","value":{"citation_required":true}},{"type":"llm-rubric","value":"Answer only from retrieved HR policy context."}]},"promptId":"prompt-hr-policy-v3","provider":{"id":"openai:gpt-4.1-mini"},"prompt":"Answer the employee policy question using only the retrieved snippets.","vars":{"question":"Can a US employee carry over unused PTO into next year?"},"response":{"output":"Employees may carry over up to 40 hours of unused PTO, subject to local policy.","tokenUsage":{"total":91,"prompt":63,"completion":28},"cost":0.00024,"latencyMs":1840},"success":true,"score":1,"latencyMs":1840,"gradingResult":{"pass":true,"score":1,"reason":"The answer is grounded and includes the required carryover limit."},"namedScores":{"faithfulness":1,"citation_span_accuracy":1},"cost":0.00024,"metadata":{"pack_id":"corp_kb_policy_qa_v1","pluginId":"policy-grounding"},"tokenUsage":{"total":91,"prompt":63,"completion":28},"evaluationId":"eval-hr-policy-20260622","traceId":"trace-hr-policy-017"},"adapter_expectations":{"maps_to":["eval_result_v1"],"required_native_fields":["id","testIdx","testCase","provider","response","success","score","gradingResult","evaluationId","traceId"],"optional_native_fields":["namedScores","cost","tokenUsage","metadata","latencyMs"],"normalization_notes":"Use evaluationId as run lineage, id as native result key, traceId as trace_ref candidate, score/gradingResult as metric evidence, and preserve full row behind harness.native_export_ref.","known_limitations":["Source-observed interface fixture, not a CLI-generated run artifact.","Promptfoo has no standalone versioned JSON Schema for this export."]},"raw_payload_hash":null,"created_at_utc":"2026-06-22T10:45:00Z"}
{"schema_version":"native_eval_fixture_v1","fixture_id":"native_openevals_evaluator_result_source_observed_001","framework":"OpenEvals","framework_version":"source-main-2026-06-22","fixture_type":"source_observed_contract","native_grain":"EvaluatorResult return object","primary_source_refs":["https://github.com/langchain-ai/openevals/blob/main/python/openevals/types.py","https://github.com/langchain-ai/openevals/blob/main/python/openevals/utils.py","https://docs.langchain.com/langsmith/openevals"],"source_retrieved_at":"2026-06-22","payload":{"key":"correctness","score":0.83,"comment":"The answer is directionally correct but omits the policy exception for union employees.","metadata":{"evaluator":"llm_as_judge","rubric_version":"policy_correctness_v2","__ls_framework":"openevals","__ls_evaluator":"create_llm_as_judge","__ls_language":"python"},"source_run_id":"run-9bb73e"},"adapter_expectations":{"maps_to":["eval_result_v1"],"required_native_fields":["key","score"],"optional_native_fields":["comment","metadata","source_run_id"],"normalization_notes":"Map key to metric id, score to normalized metric score, comment to reason_ref or inline rationale artifact, metadata to evaluator metadata, and source_run_id to trace/run foreign key.","known_limitations":["LangSmith feedback/run serialization must be captured separately when OpenEvals logs into LangSmith.","TypeScript package output shape still needs a parallel pinned fixture."]},"raw_payload_hash":null,"created_at_utc":"2026-06-22T10:45:00Z"}
{"schema_version":"native_eval_fixture_v1","fixture_id":"native_lighteval_result_bundle_source_observed_001","framework":"Hugging Face LightEval","framework_version":"source-main-2026-06-22","fixture_type":"source_observed_contract","native_grain":"aggregate result JSON plus detail artifact contract","primary_source_refs":["https://huggingface.co/docs/lighteval/en/saving-and-reading-results","https://github.com/huggingface/lighteval/blob/main/src/lighteval/logging/evaluation_tracker.py","https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/requests.py","https://github.com/huggingface/lighteval/blob/main/src/lighteval/models/model_output.py"],"source_retrieved_at":"2026-06-22","payload":{"config_general":{"model_name":"hf-internal-testing/tiny-random-gpt2","backend":"transformers","precision":"float16"},"results":{"policy_qa|0":{"exact_match":0.75,"quasi_exact_match":0.82}},"versions":{"policy_qa":"1.0"},"config_tasks":{"policy_qa|0":{"suite":["custom"],"metric":["exact_match"],"fewshot":0}},"summary_tasks":{"policy_qa|0":{"num_samples":4}},"summary_general":{"total_tasks":1},"detail_artifact_layout":{"results_path":"results/{model_name}/results_{timestamp}.json","details_path":"details/{model_name}/{timestamp}/"},"sample_detail_contract":{"doc":{"query":"What is the PTO carryover limit?","choices":["20 hours","40 hours","Unlimited"],"gold_index":1,"task_name":"policy_qa","id":"policy_017","generation_size":32,"stop_sequences":["\n"]},"model_response":{"input":"What is the PTO carryover limit?","input_tokens":[101,102],"text":["40 hours"],"output_tokens":[201,202],"text_post_processed":"40 hours","truncated_tokens_count":0,"padded_tokens_count":0}}},"adapter_expectations":{"maps_to":["eval_result_v1"],"required_native_fields":["config_general","results","versions","config_tasks","summary_tasks","summary_general"],"optional_native_fields":["detail_artifact_layout","sample_detail_contract"],"normalization_notes":"Normalize aggregate metrics into model-candidate eval results and retain detail Parquet/JSON paths as artifacts. Treat sample detail rows as evidence only when linked to a repo eval case.","known_limitations":["This fixture reflects documented/source-observed layout, not a package-generated artifact.","The separate native_lighteval_logger_bundle_real_run_001 fixture confirms logger-level detail Parquet columns doc, metric, and model_response.","A full CLI run over a real model/dataset is still needed to confirm populated config_tasks and backend-specific export variations."]},"raw_payload_hash":null,"created_at_utc":"2026-06-22T10:45:00Z"}
{"schema_version":"native_eval_fixture_v1","fixture_id":"native_deepeval_llm_test_case_source_observed_001","framework":"DeepEval","framework_version":"docs-2026-06-22","fixture_type":"source_observed_contract","native_grain":"LLMTestCase plus tool and retrieval context fields","primary_source_refs":["https://deepeval.com/docs/evaluation-test-cases"],"source_retrieved_at":"2026-06-22","payload":{"input":"Can a US employee carry over unused PTO into next year?","actual_output":"Employees may carry over up to 40 hours of unused PTO.","expected_output":"US employees may carry over up to 40 hours unless a local policy or bargaining agreement says otherwise.","context":["HR PTO policy section 4.2 states the 40 hour carryover rule."],"retrieval_context":["Confluence HR PTO section 4.2: Employees may carry over up to 40 hours."],"tools_called":[{"name":"kb.search","description":"Searches approved HR policy knowledge base","reasoning":"Policy answer requires current HR source text.","output":"Confluence HR PTO section 4.2","input_parameters":{"query":"PTO carryover US employees"}}],"expected_tools":[{"name":"kb.search","input_parameters":{"query":"PTO carryover US employees"}}],"token_cost":0.00031,"completion_time":2.14},"adapter_expectations":{"maps_to":["eval_case_v1","eval_result_v1"],"required_native_fields":["input","actual_output"],"optional_native_fields":["expected_output","context","retrieval_context","tools_called","expected_tools","token_cost","completion_time"],"normalization_notes":"Map input/expected_output to case fields, actual_output to attempt artifact, context to reference evidence, retrieval_context to retrieved evidence, and tools_called/expected_tools to tool-action failure evidence.","known_limitations":["This fixture pins the test-case object, not DeepEval's local JSON report/export schema."]},"raw_payload_hash":null,"created_at_utc":"2026-06-22T10:55:00Z"}
{"schema_version":"native_eval_fixture_v1","fixture_id":"native_ragas_single_turn_sample_source_observed_001","framework":"RAGAS","framework_version":"docs-2026-06-22","fixture_type":"source_observed_contract","native_grain":"SingleTurnSample plus EvaluationResult boundary","primary_source_refs":["https://docs.ragas.io/en/stable/references/evaluation_schema/"],"source_retrieved_at":"2026-06-22","payload":{"single_turn_sample":{"user_input":"Can a US employee carry over unused PTO into next year?","retrieved_contexts":["Confluence HR PTO section 4.2: Employees may carry over up to 40 hours."],"reference_contexts":["HR PTO policy section 4.2 states the 40 hour carryover rule and local exceptions."],"retrieved_context_ids":["chunk-pto-4-2"],"reference_context_ids":["policy-hr-pto-4-2"],"response":"Employees may carry over up to 40 hours of unused PTO.","reference":"US employees may carry over up to 40 hours unless local policy or bargaining agreement differs.","rubric":{"faithfulness":"Answer must be supported by retrieved contexts.","citation_span_accuracy":"Cited span must contain the carryover rule."},"persona_name":"hr_business_partner","query_style":"policy_lookup","query_length":"short"},"evaluation_result_boundary":{"scores":[{"faithfulness":0.92,"context_recall":0.88}],"binary_columns":[],"traces":"provider-dependent"}},"adapter_expectations":{"maps_to":["eval_case_v1","eval_result_v1"],"required_native_fields":["single_turn_sample.user_input","single_turn_sample.retrieved_contexts","single_turn_sample.response"],"optional_native_fields":["single_turn_sample.reference_contexts","single_turn_sample.retrieved_context_ids","single_turn_sample.reference_context_ids","single_turn_sample.reference","single_turn_sample.rubric","evaluation_result_boundary.scores","evaluation_result_boundary.traces"],"normalization_notes":"Keep retrieved_contexts and reference_contexts separate. Use sample fields for case/evidence construction and EvaluationResult scores only as metric output evidence.","known_limitations":["Exact serialized dataframe/JSON result shape and metric-required-field matrix still require real package-pinned fixtures."]},"raw_payload_hash":null,"created_at_utc":"2026-06-22T10:55:00Z"}
{"schema_version":"native_eval_fixture_v1","fixture_id":"native_langsmith_run_format_source_observed_001","framework":"LangSmith","framework_version":"docs-2026-06-22","fixture_type":"source_observed_contract","native_grain":"run/span data format and dataset example link","primary_source_refs":["https://docs.langchain.com/langsmith/run-data-format","https://docs.langchain.com/langsmith/example-data-format","https://docs.langchain.com/langsmith/evaluation"],"source_retrieved_at":"2026-06-22","payload":{"id":"run-9bb73e","trace_id":"trace-hr-policy-017","parent_run_id":null,"name":"hr_policy_agent","run_type":"chain","inputs":{"question":"Can a US employee carry over unused PTO into next year?"},"outputs":{"answer":"Employees may carry over up to 40 hours of unused PTO."},"events":[{"name":"start","time":"2026-06-22T10:00:01Z"},{"name":"end","time":"2026-06-22T10:00:03Z"}],"tags":["eval","hr-policy"],"feedback_stats":{"correctness":{"avg":0.83,"count":1}},"reference_example_id":"example-policy-017","session_id":"experiment-hr-policy-20260622","in_dataset":true,"total_tokens":191,"total_cost":0.00042},"adapter_expectations":{"maps_to":["trace_ref_v1","eval_result_v1"],"required_native_fields":["id","trace_id","inputs","outputs"],"optional_native_fields":["parent_run_id","events","tags","feedback_stats","reference_example_id","session_id","in_dataset","total_tokens","total_cost"],"normalization_notes":"Use LangSmith run IDs as foreign keys. Preserve run tree as trace evidence; map feedback_stats only after evaluator key and source feedback object are captured.","known_limitations":["Full serialized feedback object, annotation queue export, and experiment export edge cases still need pinned export fixtures."]},"raw_payload_hash":null,"created_at_utc":"2026-06-22T10:55:00Z"}
{"schema_version":"native_eval_fixture_v1","fixture_id":"native_phoenix_openinference_span_source_observed_001","framework":"Phoenix / OpenInference","framework_version":"docs-2026-06-22","fixture_type":"source_observed_contract","native_grain":"OpenTelemetry/OpenInference trace span plus eval output boundary","primary_source_refs":["https://arize.com/docs/phoenix/tracing/concepts-tracing/what-are-traces","https://arize.com/docs/phoenix/evaluation/llm-evals"],"source_retrieved_at":"2026-06-22","payload":{"trace":{"trace_id":"trace-hr-policy-017","span_id":"span-generation-001","parent_span_id":"span-agent-000","name":"llm.generate","start_time":"2026-06-22T10:00:01Z","end_time":"2026-06-22T10:00:03Z","status":{"code":"OK"},"attributes":{"openinference.span.kind":"LLM","llm.model_name":"gpt-4.1-mini","input.value":"Can a US employee carry over unused PTO into next year?","output.value":"Employees may carry over up to 40 hours of unused PTO.","llm.token_count.total":191},"events":[],"links":[],"resource":{"service.name":"hr-policy-agent"}},"eval_output":{"label":"correct","score":0.92,"explanation":"The answer is supported by the retrieved PTO policy span.","metadata":{"evaluator":"llm_as_judge","rubric":"grounding_v3"}}},"adapter_expectations":{"maps_to":["trace_ref_v1","eval_result_v1"],"required_native_fields":["trace.trace_id","trace.span_id","trace.name","trace.start_time","trace.end_time","trace.attributes"],"optional_native_fields":["trace.status","trace.events","trace.links","trace.resource","eval_output.label","eval_output.score","eval_output.explanation"],"normalization_notes":"Normalize the trace as trace_ref_v1 with OpenTelemetry/OpenInference semantic convention metadata. Store eval output as score evidence only when tied to an evaluator config and dataset row.","known_limitations":["Phoenix dataset, experiment, annotation, and SDK eval result export JSON still need pinned capture."]},"raw_payload_hash":null,"created_at_utc":"2026-06-22T10:55:00Z"}
{"schema_version":"native_eval_fixture_v1","fixture_id":"native_statebench_contract_validator_real_run_001","framework":"StateBench contract validator","framework_version":"repo-2026-06-22","fixture_type":"real_run_export","native_grain":"validator CLI result for native fixture contract","primary_source_refs":["statebench/runners/contract_validator.py","statebench/schemas/native_eval_fixture_v1.schema.json","statebench/fixtures/enterprise-eval-contract/v1/native-fixtures.pass.jsonl"],"source_retrieved_at":"2026-06-22","payload":{"command":"scripts/.venv-tts/bin/python3 -m statebench.runners.contract_validator --schema native_eval_fixture_v1 --input statebench/fixtures/enterprise-eval-contract/v1/native-fixtures.pass.jsonl --jsonl","exit_code":0,"stdout":"ok — statebench/fixtures/enterprise-eval-contract/v1/native-fixtures.pass.jsonl conforms to native_eval_fixture_v1","stderr":"","runner":"statebench.runners.contract_validator","schema":"native_eval_fixture_v1","input":"statebench/fixtures/enterprise-eval-contract/v1/native-fixtures.pass.jsonl"},"adapter_expectations":{"maps_to":["eval_result_v1"],"required_native_fields":["command","exit_code","stdout","runner","schema","input"],"optional_native_fields":["stderr"],"normalization_notes":"This is a repo-native real-run export proving the validator accepted the native fixture file. It is not evidence that external vendor CLIs produced their payloads; vendor package real-run fixtures remain separate work.","known_limitations":["Covers StateBench validator behavior only.","External framework package-pinned exports still need capture when dependencies are available."]},"raw_payload_hash":"sha256:5d1f8f5a29bc59c82db356abbd230ae14520f0d3c8d860880545e46f4d41a3c3","created_at_utc":"2026-06-22T11:05:00Z"}
{"schema_version":"native_eval_fixture_v1","fixture_id":"native_promptfoo_json_export_real_run_001","framework":"Promptfoo","framework_version":"0.121.17","fixture_type":"real_run_export","native_grain":"CLI JSON export from promptfoo eval","primary_source_refs":["https://www.promptfoo.dev/docs/usage/command-line/","https://github.com/promptfoo/promptfoo/blob/main/src/types/index.ts","statebench/fixtures/enterprise-eval-contract/v1/promptfoo-realrun-0.121.17.json"],"source_retrieved_at":"2026-06-22","payload":{"command":"npx --yes promptfoo@0.121.17 eval --config promptfooconfig.yaml --output /private/tmp/stateofai-promptfoo-realrun/promptfoo-realrun.json --no-progress-bar","exit_code":0,"package":"promptfoo","package_version":"0.121.17","node_version":"v23.1.0","provider":"echo","raw_export_ref":"statebench/fixtures/enterprise-eval-contract/v1/promptfoo-realrun-0.121.17.json","raw_export_sha256":"sha256:2598c5055e126ea4415823b708004a111f959da6b274ae76bc3e6994c3dd7016","native_top_level_fields":["evalId","results","config","shareableUrl","metadata","vars","runtimeOptions"]},"adapter_expectations":{"maps_to":["eval_result_v1"],"required_native_fields":["evalId","results.version","results.results","results.stats","config","metadata.promptfooVersion"],"optional_native_fields":["shareableUrl","vars","runtimeOptions","results.prompts"],"normalization_notes":"This is an external package-pinned real-run JSON export. Use results.results rows as native Promptfoo EvaluateResult evidence and retain the whole JSON export behind harness.native_export_ref.","known_limitations":["Uses the local echo provider, so it validates export shape without exercising remote model providers.","Does not cover Promptfoo red-team report exports or JSONL output."]},"raw_payload_hash":"sha256:29700a7a7fb9747131104ca7bc1db19215312176bbfa97deac1a467bc61ef944","created_at_utc":"2026-06-22T11:10:00Z"}
{"schema_version":"native_eval_fixture_v1","fixture_id":"native_promptfoo_jsonl_export_real_run_001","framework":"Promptfoo","framework_version":"0.121.17","fixture_type":"real_run_export","native_grain":"CLI JSONL per-result export from promptfoo eval","primary_source_refs":["https://www.promptfoo.dev/docs/usage/command-line/","https://github.com/promptfoo/promptfoo/blob/main/src/types/index.ts","statebench/fixtures/enterprise-eval-contract/v1/promptfoo-realrun-0.121.17.jsonl"],"source_retrieved_at":"2026-06-22","payload":{"command":"npx --yes promptfoo@0.121.17 eval --config promptfooconfig.yaml --output /private/tmp/stateofai-promptfoo-realrun/promptfoo-realrun.jsonl --no-progress-bar","exit_code":0,"package":"promptfoo","package_version":"0.121.17","node_version":"v23.1.0","provider":"echo","raw_export_ref":"statebench/fixtures/enterprise-eval-contract/v1/promptfoo-realrun-0.121.17.jsonl","raw_export_sha256":"sha256:ee73544e22e688ae028bedfd8f9e7303d01323759051d2560e109c4314726357","native_top_level_fields":["cost","gradingResult","id","latencyMs","prompt","promptId","promptIdx","provider","response","score","success","testCase","testIdx","tokenUsage","vars"]},"adapter_expectations":{"maps_to":["eval_result_v1"],"required_native_fields":["id","testIdx","testCase","provider","response","success","score","gradingResult"],"optional_native_fields":["cost","latencyMs","namedScores","promptId","tokenUsage","vars","metadata","failureReason"],"normalization_notes":"This is an external package-pinned real-run JSONL export. Treat each JSONL row as one Promptfoo EvaluateResult-style native result row and retain the raw JSONL export behind harness.native_export_ref.","known_limitations":["Uses the local echo provider, so it validates per-result export shape without exercising remote model providers.","Does not cover Promptfoo red-team report exports."]},"raw_payload_hash":"sha256:167c3a2f65f003f8b7a34852ff70a4de178db4965202c931f0a85d766b384391","created_at_utc":"2026-06-22T11:15:00Z"}
{"schema_version":"native_eval_fixture_v1","fixture_id":"native_ragas_dataset_jsonl_real_run_001","framework":"RAGAS","framework_version":"0.4.3","fixture_type":"real_run_export","native_grain":"EvaluationDataset.to_jsonl output for a SingleTurnSample","primary_source_refs":["https://docs.ragas.io/en/stable/references/evaluation_schema/","statebench/fixtures/enterprise-eval-contract/v1/ragas-dataset-realrun-0.4.3.jsonl"],"source_retrieved_at":"2026-06-22","payload":{"command":"PYTHONPATH=/private/tmp/stateofai-ragas-realrun/pkg scripts/.venv-tts/bin/python3 generate_ragas_dataset_to_jsonl","dataset_features":["user_input","retrieved_contexts","reference_contexts","retrieved_context_ids","reference_context_ids","response","reference","rubrics","persona_name","query_style","query_length"],"exit_code":0,"native_top_level_fields":["user_input","retrieved_contexts","reference_contexts","retrieved_context_ids","reference_context_ids","response","reference","rubrics","persona_name","query_style","query_length"],"package":"ragas","package_version":"0.4.3","python_version":"3.13","raw_export_ref":"statebench/fixtures/enterprise-eval-contract/v1/ragas-dataset-realrun-0.4.3.jsonl","raw_export_sha256":"sha256:6dcf1e8ab40c1fd54928c4bc0ca1a3437314b536ac57cfbc3f0c6e6bd7b627b8","serialization_api":"EvaluationDataset.to_jsonl"},"adapter_expectations":{"maps_to":["eval_case_v1","eval_result_v1"],"required_native_fields":["user_input","retrieved_contexts","response"],"optional_native_fields":["reference_contexts","retrieved_context_ids","reference_context_ids","reference","rubrics","persona_name","query_style","query_length"],"normalization_notes":"This is an external package-pinned RAGAS dataset JSONL serialization fixture. Treat the JSONL row as sample/case evidence; do not treat it as a scored EvaluationResult.","known_limitations":["Covers EvaluationDataset.to_jsonl and SingleTurnSample serialization only.","Does not cover EvaluationResult scores, cost callback, traces, ragas_traces, or metric-specific required fields.","Top-level ragas import in this isolated install attempted an optional VertexAI path; this fixture loads ragas.dataset_schema directly from the installed package to exercise the serialization class without model or provider dependencies."]},"raw_payload_hash":"sha256:b63c3702be763d5f7c1de35b9144bbf6deab945b1ff9ce1370da3fc5d42e3afb","created_at_utc":"2026-06-22T11:35:00Z"}
{"schema_version":"native_eval_fixture_v1","fixture_id":"native_huggingface_evaluate_save_real_run_001","framework":"Hugging Face Evaluate","framework_version":"0.4.6","fixture_type":"real_run_export","native_grain":"evaluate.save JSON result artifact","primary_source_refs":["https://huggingface.co/docs/evaluate/package_reference/main_classes","statebench/fixtures/enterprise-eval-contract/v1/evaluate-realrun-0.4.6.json"],"source_retrieved_at":"2026-06-22","payload":{"command":"PYTHONPATH=/private/tmp/stateofai-evaluate-realrun/pkg scripts/.venv-tts/bin/python3 generate_evaluate_save_fixture","exit_code":0,"metric_name":"accuracy","metric_result_fields":["accuracy"],"native_top_level_fields":["experiment","framework_fixture","metric_name","predictions","references","accuracy","_timestamp","_git_commit_hash","_evaluate_version","_python_version","_interpreter_path"],"package":"evaluate","package_version":"0.4.6","raw_export_ref":"statebench/fixtures/enterprise-eval-contract/v1/evaluate-realrun-0.4.6.json","raw_export_sha256":"sha256:395ef6000050b14b91b265d992f01ffc043d3fb3d34ef9208bbc08feef7a76ea","serialization_api":"evaluate.save"},"adapter_expectations":{"maps_to":["eval_result_v1"],"required_native_fields":["metric_name","accuracy","_evaluate_version","_timestamp"],"optional_native_fields":["experiment","framework_fixture","predictions","references","_git_commit_hash","_python_version","_interpreter_path"],"normalization_notes":"This is an external package-pinned Hugging Face Evaluate serialization fixture. Treat the JSON as a metric result artifact with Evaluate runtime metadata; normalize metric-specific keys only when a per-metric adapter declares them.","known_limitations":["Covers evaluate.save serialization only, not evaluate.load metric-module execution.","Accuracy was supplied as a deterministic offline result to avoid Hugging Face Hub downloads.","Does not exercise model, dataset, distributed cache, or EvaluationSuite execution paths."]},"raw_payload_hash":"sha256:06edc659fd7b629af122a8fcff5f9feffdeb3302e5732e41660011082962dc1e","created_at_utc":"2026-06-22T11:45:00Z"}
{"schema_version":"native_eval_fixture_v1","fixture_id":"native_deepeval_test_run_real_run_001","framework":"DeepEval","framework_version":"4.0.6","fixture_type":"real_run_export","native_grain":"offline deepeval.evaluate TestRun JSON","primary_source_refs":["https://deepeval.com/docs/getting-started","https://deepeval.com/docs/evaluation-test-cases","https://deepeval.com/docs/metrics-introduction","statebench/fixtures/enterprise-eval-contract/v1/deepeval-test-run-realrun-4.0.6.json"],"source_retrieved_at":"2026-06-22","payload":{"command":"PYTHONPATH=/private/tmp/stateofai-deepeval-realrun/pkg DEEPEVAL_DISABLE_DOTENV=1 DEEPEVAL_RESULTS_FOLDER=/private/tmp/stateofai-deepeval-realrun/out /Users/pierre/consulting/stateofai/scripts/.venv-tts/bin/python3 generate_deepeval_offline_test_run","exit_code":0,"metric_data_fields":["name","threshold","success","score","reason","strictMode","evaluationModel","evaluationCost","verboseLogs"],"metric_name":"Exact Contains","native_top_level_fields":["testCases","conversationalTestCases","metricsScores","identifier","prompts","testPassed","testFailed","runDuration","evaluationCost","official"],"package":"deepeval","package_version":"4.0.6","raw_export_ref":"statebench/fixtures/enterprise-eval-contract/v1/deepeval-test-run-realrun-4.0.6.json","raw_export_sha256":"sha256:ec4726ef80b292e7bc908e2742735a6774cb88b5608c036a95a460816d956bef","serialization_api":"deepeval.evaluate local TestRun JSON","test_case_fields":["name","input","actualOutput","expectedOutput","context","retrievalContext","tags","success","metricsData","runDuration","evaluationCost","order"]},"adapter_expectations":{"maps_to":["eval_result_v1","eval_case_v1"],"required_native_fields":["testCases","metricsScores","testPassed","testFailed","identifier"],"optional_native_fields":["conversationalTestCases","prompts","runDuration","evaluationCost","official"],"normalization_notes":"This is an external package-pinned DeepEval offline TestRun JSON generated with a deterministic custom BaseMetric. Map testCases rows to eval attempts and metricsData/metricsScores to normalized metric scores while retaining the raw TestRun artifact.","known_limitations":["Covers local deepeval.evaluate TestRun serialization only, not Confident AI hosted exports or component trace export shapes.","Uses a deterministic custom BaseMetric to avoid API keys; built-in LLM metrics and judge models still need separate pinned fixtures.","DeepEval attempted PostHog telemetry after the successful run and DNS failed, but the process exit code remained 0.","Run was executed from /private/tmp to avoid committing DeepEval's .deepeval/.latest_run_full.json side effect."]},"raw_payload_hash":"sha256:521c9b565860bb75cc079e9b50c75166f7aa9a46f7ad097ce538718c0229edc5","created_at_utc":"2026-06-22T11:55:00Z"}
{"schema_version":"native_eval_fixture_v1","fixture_id":"native_lighteval_logger_bundle_real_run_001","framework":"Hugging Face LightEval","framework_version":"0.13.0","fixture_type":"real_run_export","native_grain":"EvaluationTracker.save aggregate JSON plus detail parquet bundle manifest","primary_source_refs":["https://huggingface.co/docs/lighteval/en/saving-and-reading-results","https://github.com/huggingface/lighteval/blob/main/src/lighteval/logging/evaluation_tracker.py","https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/requests.py","https://github.com/huggingface/lighteval/blob/main/src/lighteval/models/model_output.py","statebench/fixtures/enterprise-eval-contract/v1/lighteval-logger-realrun-0.13.0.json"],"source_retrieved_at":"2026-06-22","payload":{"command":"PYTHONPATH=/private/tmp/stateofai-lighteval-realrun/pkg scripts/.venv-tts/bin/python3 generate_lighteval_logger_fixture","detail_parquet_fields":["doc","metric","model_response"],"exit_code":0,"native_top_level_fields":["config_general","results","versions","config_tasks","summary_tasks","summary_general"],"output_bundle_files":["results/{model_name}/results_{timestamp}.json","details/{model_name}/{timestamp}/details_{task_name}_{timestamp}.parquet"],"package":"lighteval","package_version":"0.13.0","raw_export_ref":"statebench/fixtures/enterprise-eval-contract/v1/lighteval-logger-realrun-0.13.0.json","raw_export_sha256":"sha256:1b85e2c0ebfbb485d5bf945782c5b256948717b142fb2c56ab0b03d76a671304","serialization_api":"EvaluationTracker.save"},"adapter_expectations":{"maps_to":["eval_result_v1"],"required_native_fields":["config_general","results","versions","summary_tasks","summary_general"],"optional_native_fields":["config_tasks","detail_parquet_fields","output_bundle_files"],"normalization_notes":"This is an external package-pinned LightEval logger-level bundle fixture generated through EvaluationTracker.save. Normalize aggregate metrics from results and retain the detail parquet bundle as sample evidence when linked to a repo eval case.","known_limitations":["Covers LightEval EvaluationTracker.save serialization and observed detail parquet columns only.","Does not run the full LightEval CLI over a downloaded model or dataset.","config_tasks is empty in this logger-level fixture because no LightevalTask object was loaded.","LightEval was installed into an isolated target directory; pip reported dependency conflicts against the ambient environment, but imports and logger save completed under PYTHONPATH."]},"raw_payload_hash":"sha256:39b85c43cdbdc161d5af40c3e51c2ffb0f0e95df602c08ce8b9840f4c99c24ee","created_at_utc":"2026-06-22T12:10:00Z"}