{ "schema_version": "eval_native_schema_boundaries_2026_v1", "retrieved_at": "2026-06-22", "purpose": "Adapter-facing native schema boundary ledger for enterprise eval ingestion. These records distinguish documented public contracts from source-observed implementation shapes.", "fixture_contract": { "schema": "statebench/schemas/native_eval_fixture_v1.schema.json", "passing_fixture_file": "statebench/fixtures/enterprise-eval-contract/v1/native-fixtures.pass.jsonl", "gap_schema": "statebench/schemas/eval_source_gap_v1.schema.json", "gap_backlog_file": "sources/21-benchmarks/eval-native-schema-gap-backlog-2026.jsonl", "gap_backlog_validator": "statebench/runners/validate_eval_gap_backlog.py", "smoke_command": "scripts/.venv-tts/bin/python3 -m statebench.runners.enterprise_eval_contract_smoke", "validated_frameworks": [ "DeepEval", "RAGAS", "LangSmith", "Phoenix / OpenInference", "Promptfoo", "OpenEvals", "Hugging Face Evaluate", "Hugging Face LightEval", "StateBench contract validator" ], "external_real_run_exports": [ { "framework": "DeepEval", "version": "4.0.6", "serialization_api": "deepeval.evaluate local TestRun JSON", "raw_export_ref": "statebench/fixtures/enterprise-eval-contract/v1/deepeval-test-run-realrun-4.0.6.json", "raw_export_sha256": "sha256:ec4726ef80b292e7bc908e2742735a6774cb88b5608c036a95a460816d956bef" }, { "framework": "Promptfoo", "version": "0.121.17", "provider": "echo", "raw_export_ref": "statebench/fixtures/enterprise-eval-contract/v1/promptfoo-realrun-0.121.17.json", "raw_export_sha256": "sha256:2598c5055e126ea4415823b708004a111f959da6b274ae76bc3e6994c3dd7016" }, { "framework": "Promptfoo", "version": "0.121.17", "provider": "echo", "raw_export_ref": "statebench/fixtures/enterprise-eval-contract/v1/promptfoo-realrun-0.121.17.jsonl", "raw_export_sha256": "sha256:ee73544e22e688ae028bedfd8f9e7303d01323759051d2560e109c4314726357" }, { "framework": "RAGAS", "version": "0.4.3", "serialization_api": "EvaluationDataset.to_jsonl", "raw_export_ref": "statebench/fixtures/enterprise-eval-contract/v1/ragas-dataset-realrun-0.4.3.jsonl", "raw_export_sha256": "sha256:6dcf1e8ab40c1fd54928c4bc0ca1a3437314b536ac57cfbc3f0c6e6bd7b627b8" }, { "framework": "Hugging Face Evaluate", "version": "0.4.6", "serialization_api": "evaluate.save", "raw_export_ref": "statebench/fixtures/enterprise-eval-contract/v1/evaluate-realrun-0.4.6.json", "raw_export_sha256": "sha256:395ef6000050b14b91b265d992f01ffc043d3fb3d34ef9208bbc08feef7a76ea" }, { "framework": "Hugging Face LightEval", "version": "0.13.0", "serialization_api": "EvaluationTracker.save logger-level bundle manifest", "raw_export_ref": "statebench/fixtures/enterprise-eval-contract/v1/lighteval-logger-realrun-0.13.0.json", "raw_export_sha256": "sha256:1b85e2c0ebfbb485d5bf945782c5b256948717b142fb2c56ab0b03d76a671304" } ], "fixture_types": [ "source_observed_contract", "documentation_example", "real_run_export" ], "boundary_note": "Current external-framework native fixtures are source-observed contracts, not package-pinned real-run exports unless a row declares fixture_type=real_run_export. Real-run fixture rows must carry a sha256 payload hash and are checked by statebench.runners.contract_validator." }, "rollup_contracts": { "team_schema": "statebench/schemas/team_eval_rollup_v1.schema.json", "team_fixture": "statebench/fixtures/enterprise-eval-contract/v1/team-rollup.pass.json", "firm_schema": "statebench/schemas/firm_eval_rollup_v1.schema.json", "firm_fixture": "statebench/fixtures/enterprise-eval-contract/v1/firm-rollup.pass.json", "rollup_collector": "statebench/runners/eval_rollup_collector.py", "gitlab_component_template": "statebench/ci/gitlab-enterprise-eval-component.yml", "gitlab_component_validator": "statebench/runners/validate_gitlab_eval_component.py", "warehouse_tables": [ "team_eval_rollups", "firm_eval_rollups" ], "boundary_note": "Repo-local eval_result_v1 rows roll up into team_eval_rollup_v1 windows; team rollups roll up into firm_eval_rollup_v1 windows for governance, scorecards, knowledge promotions, and training candidate counts." }, "normalized_result_contract": { "schema": "statebench/schemas/eval_result_v1.schema.json", "pass_fixture": "statebench/fixtures/enterprise-eval-contract/v1/eval-results.pass.jsonl", "fail_fixture": "statebench/fixtures/enterprise-eval-contract/v1/eval-results.fail.jsonl", "required_identity_fields": [ "harness.name", "harness.version", "harness.runner_kind", "model.provider", "model.name", "model.version", "runtime_context.prompt_template_version", "runtime_context.retrieval_index_version", "runtime_context.tool_catalog_version", "runtime_context.judge_model_version", "runtime_context.execution_environment", "attached_tools[].tool_id", "attached_tools[].tool_type", "attached_tools[].name", "attached_tools[].version", "attached_tools[].source" ], "boundary_note": "Every normalized eval_result_v1 row must preserve the runner kind/version, model version, judge/runtime context, and attached tool inventory. Tool rows can also carry schema refs, schema hashes, permission profiles, network access class, and data scope so team and firm collectors can classify failures against the exact harness surface that produced them." }, "records": [ { "framework": "DeepEval", "native_grain": "LLMTestCase and local deepeval.evaluate TestRun JSON", "primary_sources": [ "https://deepeval.com/docs/getting-started", "https://deepeval.com/docs/evaluation-test-cases", "https://deepeval.com/docs/metrics-introduction", "statebench/fixtures/enterprise-eval-contract/v1/deepeval-test-run-realrun-4.0.6.json" ], "stable_fields": { "llm_test_case": [ "input", "actual_output", "expected_output", "context", "retrieval_context", "tools_called", "expected_tools", "token_cost", "completion_time" ], "test_run": [ "testCases", "conversationalTestCases", "metricsScores", "identifier", "prompts", "testPassed", "testFailed", "runDuration", "evaluationCost", "official" ], "test_case_result": [ "name", "input", "actualOutput", "expectedOutput", "context", "retrievalContext", "tags", "success", "metricsData", "runDuration", "evaluationCost", "order" ], "metric_data": [ "name", "threshold", "success", "score", "reason", "strictMode", "evaluationModel", "evaluationCost", "verboseLogs" ] }, "adapter_mapping": "Map TestRun testCases to eval attempts, context/retrievalContext to reference and retrieved evidence, and metricsData/metricsScores to eval_result_v1 scores. Retain the raw TestRun JSON as the native export reference.", "remaining_gaps": [ "Package-pinned local no-key TestRun JSON is captured for deepeval==4.0.6", "Hosted Confident AI export shape still needs capture", "Component trace export shape still needs capture", "Built-in LLM judge metrics still need pinned fixtures with model/provider metadata" ] }, { "framework": "Hugging Face Evaluate", "native_grain": "metric result dictionary and evaluate.save JSON artifact", "primary_sources": [ "https://huggingface.co/docs/evaluate/package_reference/main_classes", "https://huggingface.co/docs/evaluate/package_reference/evaluator_classes", "statebench/fixtures/enterprise-eval-contract/v1/evaluate-realrun-0.4.6.json" ], "stable_fields": { "evaluate_save_artifact": [ "experiment", "framework_fixture", "metric_name", "predictions", "references", "accuracy", "_timestamp", "_git_commit_hash", "_evaluate_version", "_python_version", "_interpreter_path" ], "runtime_metadata": [ "_timestamp", "_git_commit_hash", "_evaluate_version", "_python_version", "_interpreter_path" ], "metric_result": [ "metric-specific keys such as accuracy" ] }, "adapter_mapping": "Store the raw evaluate.save artifact as native evidence and normalize only declared metric-specific keys into eval_result_v1 scores. Keep metric-module execution, evaluator invocations, and distributed cache metadata separate from basic result serialization.", "remaining_gaps": [ "Package-pinned evaluate.save serialization is captured for evaluate==0.4.6", "evaluate.load metric-module execution still needs selected per-metric pinned fixtures", "EvaluationSuite and distributed cache artifacts still need pinned captures" ] }, { "framework": "RAGAS", "native_grain": "SingleTurnSample, MultiTurnSample, EvaluationDataset JSONL, and scored EvaluationResult", "primary_sources": [ "https://docs.ragas.io/en/stable/references/evaluation_schema/", "statebench/fixtures/enterprise-eval-contract/v1/ragas-dataset-realrun-0.4.3.jsonl" ], "stable_fields": { "single_turn_sample": [ "user_input", "retrieved_contexts", "reference_contexts", "retrieved_context_ids", "reference_context_ids", "response", "multi_responses", "reference", "rubrics", "persona_name", "query_style", "query_length" ], "evaluation_dataset_jsonl": [ "user_input", "retrieved_contexts", "reference_contexts", "retrieved_context_ids", "reference_context_ids", "response", "reference", "rubrics", "persona_name", "query_style", "query_length" ], "evaluation_result": [ "scores", "dataset", "binary_columns", "cost_cb", "traces", "ragas_traces", "run_id" ] }, "adapter_mapping": "Map EvaluationDataset JSONL rows to eval_case_v1 case/source evidence. Map scored EvaluationResult rows to eval_result_v1 only after the metric-specific required fields and trace/cost behavior are captured for the pinned version.", "remaining_gaps": [ "Package-pinned dataset JSONL serialization is captured for ragas==0.4.3", "Scored EvaluationResult dataframe/JSON shape still needs a pinned run with valid ragas_traces", "Metric-specific required-field matrix still needs capture for selected enterprise RAG metrics" ] }, { "framework": "Promptfoo", "native_grain": "whole-run JSON bundle and per-result JSONL row", "primary_sources": [ "https://www.promptfoo.dev/docs/usage/command-line/", "https://github.com/promptfoo/promptfoo/blob/main/src/types/index.ts", "https://github.com/promptfoo/promptfoo/blob/main/src/util/output.ts" ], "stable_fields": { "output_file": [ "evalId", "results", "config", "shareableUrl", "metadata", "vars", "runtimeOptions", "traces", "blobAssets" ], "evaluate_summary_v3": [ "version", "timestamp", "results", "prompts", "stats" ], "evaluate_result": [ "id", "description", "promptIdx", "testIdx", "testCase", "promptId", "provider", "prompt", "vars", "response", "error", "failureReason", "success", "score", "latencyMs", "gradingResult", "namedScores", "cost", "metadata", "tokenUsage", "evaluationId", "traceId" ] }, "adapter_mapping": "Map JSONL rows to eval_result_v1 attempts and scores; store JSON bundle as native_export_ref. JUnit is CI annotation evidence, not canonical scoring evidence.", "remaining_gaps": [ "No standalone published JSON Schema for exports", "Separate red-team report UI/export schema needs deeper source tracing", "Version-pinned fixture needed before production adapter lock" ] }, { "framework": "OpenEvals", "native_grain": "evaluator return object", "primary_sources": [ "https://github.com/langchain-ai/openevals/blob/main/python/openevals/types.py", "https://github.com/langchain-ai/openevals/blob/main/python/openevals/utils.py", "https://docs.langchain.com/langsmith/openevals" ], "stable_fields": { "evaluator_result": [ "key", "score", "comment", "metadata", "source_run_id" ], "simple_evaluator_signature": [ "inputs", "outputs", "reference_outputs", "kwargs" ], "langsmith_feedback": [ "key", "score", "comment" ] }, "adapter_mapping": "Map key to metric_id, score to normalized score, comment to rationale, metadata/source_run_id to raw evidence and trace references.", "remaining_gaps": [ "TypeScript package object types need the same source-level pass", "LangSmith docs do not expose a full serialized feedback/run payload for OpenEvals" ] }, { "framework": "Hugging Face Evaluate", "native_grain": "metric-specific compute result and EvaluationSuite task output", "primary_sources": [ "https://huggingface.co/docs/evaluate/package_reference/evaluator_classes", "https://huggingface.co/docs/evaluate/package_reference/main_classes", "https://github.com/huggingface/evaluate/blob/main/src/evaluate/module.py", "https://github.com/huggingface/evaluate/blob/main/src/evaluate/evaluation_suite/__init__.py" ], "stable_fields": { "evaluation_suite_result_annotation": [ "task_name", "data_preprocessor" ], "distributed_cache_metadata": [ "cache_dir", "keep_in_memory", "num_process", "process_id", "experiment_id", "max_concurrent_cache_files", "timeout", "seed", "hash" ] }, "adapter_mapping": "Store metric name/config and raw result dictionary. Normalize only selected metric keys into eval_result_v1 scores after a per-metric adapter declares the key mapping.", "remaining_gaps": [ "Upstream intentionally has no universal metric result schema", "Enterprise adapters need per-metric mappings for selected metrics" ] }, { "framework": "Hugging Face LightEval", "native_grain": "model benchmark aggregate JSON plus sample detail parquet", "primary_sources": [ "https://huggingface.co/docs/lighteval/en/saving-and-reading-results", "https://github.com/huggingface/lighteval/blob/main/src/lighteval/logging/evaluation_tracker.py", "https://github.com/huggingface/lighteval/blob/main/src/lighteval/tasks/requests.py", "https://github.com/huggingface/lighteval/blob/main/src/lighteval/models/model_output.py" ], "stable_fields": { "result_json_sections": [ "config_general", "results", "versions", "config_tasks", "summary_tasks", "summary_general" ], "doc": [ "query", "choices", "gold_index", "instruction", "images", "specific", "unconditioned_query", "original_query", "id", "task_name", "fewshot_samples", "sampling_methods", "generation_size", "stop_sequences", "use_logits", "num_samples", "generation_grammar" ], "model_response": [ "input", "input_tokens", "text", "output_tokens", "text_post_processed", "reasonings", "logprobs", "argmax_logits_eq_gold", "logits", "unconditioned_logprobs", "truncated_tokens_count", "padded_tokens_count" ] }, "adapter_mapping": "Treat as model-candidate benchmark evidence. Normalize aggregate metrics and retain result JSON plus detail parquet artifact refs.", "remaining_gaps": [ "Need full LightEval CLI run over a real model/dataset to confirm task config population and backend-specific export variations", "Logger-level lighteval==0.13.0 fixture confirms result JSON sections and detail parquet columns doc, metric, and model_response" ] }, { "framework": "OpenAI Agents SDK", "native_grain": "trace and span", "primary_sources": [ "https://openai.github.io/openai-agents-python/tracing/" ], "stable_fields": { "trace": [ "workflow_name", "trace_id", "group_id", "disabled", "metadata" ], "span": [ "started_at", "ended_at", "trace_id", "parent_id", "span_data" ], "default_span_families": [ "runner", "agent", "generation", "function", "guardrail", "handoff", "transcription", "speech", "speech_group" ] }, "adapter_mapping": "Map span coverage into trace_ref_v1. Require generation, tool/function, guardrail, handoff, and final-output evidence for agent governance gates.", "remaining_gaps": [ "Hosted tracing unavailable for OpenAI API ZDR organizations", "Production adapter should capture raw processor/export payloads from a concrete trace processor" ] }, { "framework": "Google ADK", "native_grain": "runtime event, event action, invocation eval case, and eval set", "primary_sources": [ "https://adk.dev/evaluate/", "https://github.com/google/adk-python/blob/main/src/google/adk/events/event.py", "https://github.com/google/adk-python/blob/main/src/google/adk/events/event_actions.py", "https://github.com/google/adk-python/blob/main/src/google/adk/evaluation/eval_case.py", "https://github.com/google/adk-python/blob/main/src/google/adk/evaluation/eval_set.py" ], "stable_fields": { "event": [ "invocation_id", "author", "actions", "output", "node_info", "long_running_tool_ids", "branch", "isolation_scope", "id", "timestamp" ], "event_actions": [ "skip_summarization", "state_delta", "artifact_delta", "transfer_to_agent", "escalate", "requested_auth_configs", "requested_tool_confirmations", "compaction", "end_of_agent", "agent_state", "rewind_before_invocation_id", "route", "render_ui_widgets", "set_model_response" ], "eval_projection": [ "eval_set_id", "eval_cases", "invocation_id", "user_content", "final_response", "intermediate_data", "rubrics", "app_details" ] }, "adapter_mapping": "Map eval projection to eval_case_v1. Store runtime Event and EventActions payloads under trace_ref_v1 for trajectory and tool-action evidence.", "remaining_gaps": [ "Google Agent Engine platform event export shape still needs capture beyond open-source ADK" ] }, { "framework": "Microsoft AutoGen", "native_grain": "OpenTelemetry runtime/tool/agent spans and event logs", "primary_sources": [ "https://microsoft.github.io/autogen/stable/user-guide/core-user-guide/framework/telemetry.html", "https://microsoft.github.io/autogen/stable/user-guide/agentchat-user-guide/tracing.html", "https://github.com/microsoft/autogen/blob/main/python/packages/autogen-core/src/autogen_core/_single_threaded_agent_runtime.py" ], "stable_fields": { "runtime_operations": [ "create", "send", "publish", "process", "ack", "intercept" ], "runtime_attributes": [ "sender_agent_type", "sender_agent_class", "recipient_agent_type", "recipient_agent_class", "message_context", "message" ], "message_context": [ "sender", "topic_id", "is_rpc", "message_id" ], "message_event": [ "payload", "sender", "receiver", "kind", "delivery_stage" ] }, "adapter_mapping": "Normalize as OpenTelemetry trace_ref_v1 payload with semantic convention version. Use message event fields for agent handoff and delivery failure classification.", "remaining_gaps": [ "Microsoft Foundry-specific eval and telemetry exports were not verified in this pass" ] }, { "framework": "Mistral Agents and Conversations", "native_grain": "conversation response and output entry union", "primary_sources": [ "https://docs.mistral.ai/studio-api/agents/introduction", "https://docs.mistral.ai/api", "https://github.com/mistralai/client-python/tree/main/docs/models" ], "stable_fields": { "conversation_response": [ "object", "conversation_id", "outputs", "usage", "guardrails" ], "output_entry_types": [ "message.output", "tool.execution", "function.call", "agent.handoff" ], "handoff_entry": [ "previous_agent_id", "previous_agent_name", "next_agent_id", "next_agent_name" ] }, "adapter_mapping": "Normalize conversation outputs into trace_ref_v1 spans and eval_result_v1 artifact refs. Tool, function, and handoff entries map to tool-action and trajectory failure domains.", "remaining_gaps": [ "Mistral observability judge/eval object schemas need OpenAPI-level extraction" ] }, { "framework": "Qwen-Agent", "native_grain": "message stream and function-call schema", "primary_sources": [ "https://qwen.readthedocs.io/en/latest/framework/function_call.html", "https://github.com/QwenLM/Qwen-Agent/blob/main/qwen_agent/agent.py", "https://github.com/QwenLM/Qwen-Agent/blob/main/qwen_agent/agents/react_chat.py", "https://github.com/QwenLM/Qwen-Agent/blob/main/qwen_agent/llm/schema.py" ], "stable_fields": { "message": [ "role", "content", "reasoning_content", "name", "function_call", "extra" ], "function_call": [ "name", "arguments" ], "content_item_oneof": [ "text", "image", "file", "audio", "video" ], "react_stream_markers": [ "Thought", "Action", "Action Input", "Observation", "Final Answer" ] }, "adapter_mapping": "Normalize message stream into trajectory/tool-call evidence. Treat ReAct markers as source-observed text protocol, not a durable event envelope.", "remaining_gaps": [ "No first-class Qwen-Agent trace/event telemetry envelope found in primary repo files" ] } ] }