ModernJEV-Decide-Preview: model, results, compute costs and ML Intern workflow

.gitattributes

@@ -0,0 +1,35 @@
+*.7z filter=lfs diff=lfs merge=lfs -text
+*.arrow filter=lfs diff=lfs merge=lfs -text
+*.bin filter=lfs diff=lfs merge=lfs -text
+*.bz2 filter=lfs diff=lfs merge=lfs -text
+*.ckpt filter=lfs diff=lfs merge=lfs -text
+*.ftz filter=lfs diff=lfs merge=lfs -text
+*.gz filter=lfs diff=lfs merge=lfs -text
+*.h5 filter=lfs diff=lfs merge=lfs -text
+*.joblib filter=lfs diff=lfs merge=lfs -text
+*.lfs.* filter=lfs diff=lfs merge=lfs -text
+*.mlmodel filter=lfs diff=lfs merge=lfs -text
+*.model filter=lfs diff=lfs merge=lfs -text
+*.msgpack filter=lfs diff=lfs merge=lfs -text
+*.npy filter=lfs diff=lfs merge=lfs -text
+*.npz filter=lfs diff=lfs merge=lfs -text
+*.onnx filter=lfs diff=lfs merge=lfs -text
+*.ot filter=lfs diff=lfs merge=lfs -text
+*.parquet filter=lfs diff=lfs merge=lfs -text
+*.pb filter=lfs diff=lfs merge=lfs -text
+*.pickle filter=lfs diff=lfs merge=lfs -text
+*.pkl filter=lfs diff=lfs merge=lfs -text
+*.pt filter=lfs diff=lfs merge=lfs -text
+*.pth filter=lfs diff=lfs merge=lfs -text
+*.rar filter=lfs diff=lfs merge=lfs -text
+*.safetensors filter=lfs diff=lfs merge=lfs -text
+saved_model/**/* filter=lfs diff=lfs merge=lfs -text
+*.tar.* filter=lfs diff=lfs merge=lfs -text
+*.tar filter=lfs diff=lfs merge=lfs -text
+*.tflite filter=lfs diff=lfs merge=lfs -text
+*.tgz filter=lfs diff=lfs merge=lfs -text
+*.wasm filter=lfs diff=lfs merge=lfs -text
+*.xz filter=lfs diff=lfs merge=lfs -text
+*.zip filter=lfs diff=lfs merge=lfs -text
+*.zst filter=lfs diff=lfs merge=lfs -text
+*tfevents* filter=lfs diff=lfs merge=lfs -text

COST-AUDIT.json

@@ -0,0 +1,207 @@
+{
+  "source": "Live job total durations (including scheduling conservatively) and current published per-minute hardware rates; not an invoice",
+  "invoice_verified": false,
+  "previous_jobs_compute_upper_bound_usd": 2.208502,
+  "jobs": [
+    {
+      "id": "6abce8d0031314b6963447a4",
+      "flavor": "cpu-basic",
+      "startedAt": "2026-09-30T10:47:47.400Z",
+      "finishedAt": "2026-09-30T10:47:50.469Z",
+      "durations": {
+        "schedulingSecs": 3,
+        "runningSecs": 3,
+        "totalSecs": 6
+      },
+      "status": {
+        "stage": "ERROR",
+        "message": "Job failed with exit code: 3. Reason: Error",
+        "failureCount": 0
+      },
+      "conservative_billable_minutes": 1,
+      "pricing_unit_usd": 0.000167,
+      "estimated_upper_bound_usd": 0.000167
+    },
+    {
+      "id": "6abcf03c031314b69634487d",
+      "flavor": "a100-large",
+      "startedAt": "2026-09-30T11:19:38.618Z",
+      "finishedAt": "2026-09-30T11:26:45.280Z",
+      "durations": {
+        "schedulingSecs": 13,
+        "runningSecs": 426,
+        "totalSecs": 440
+      },
+      "status": {
+        "stage": "ERROR",
+        "message": "Job failed with exit code: 1. Reason: Error",
+        "failureCount": 0
+      },
+      "conservative_billable_minutes": 8,
+      "pricing_unit_usd": 0.041667,
+      "estimated_upper_bound_usd": 0.333336
+    },
+    {
+      "id": "6abcf3be4c46ef198703619a",
+      "flavor": "a100-large",
+      "startedAt": "2026-09-30T11:34:28.605Z",
+      "finishedAt": "2026-09-30T11:37:23.342Z",
+      "durations": {
+        "schedulingSecs": 5,
+        "runningSecs": 174,
+        "totalSecs": 180
+      },
+      "status": {
+        "stage": "ERROR",
+        "message": "Job failed with exit code: 1. Reason: Error",
+        "failureCount": 0
+      },
+      "conservative_billable_minutes": 3,
+      "pricing_unit_usd": 0.041667,
+      "estimated_upper_bound_usd": 0.125001
+    },
+    {
+      "id": "6abcfa794c46ef1987036338",
+      "flavor": "a100-large",
+      "startedAt": "2026-09-30T12:03:11.619Z",
+      "finishedAt": "2026-09-30T12:05:30.241Z",
+      "durations": {
+        "schedulingSecs": 6,
+        "runningSecs": 138,
+        "totalSecs": 144
+      },
+      "status": {
+        "stage": "ERROR",
+        "message": "Job failed with exit code: 1. Reason: Error",
+        "failureCount": 0
+      },
+      "conservative_billable_minutes": 3,
+      "pricing_unit_usd": 0.041667,
+      "estimated_upper_bound_usd": 0.125001
+    },
+    {
+      "id": "6abcff404c46ef1987036446",
+      "flavor": "a100-large",
+      "startedAt": "2026-09-30T12:23:44.354Z",
+      "finishedAt": "2026-09-30T12:25:32.810Z",
+      "durations": {
+        "schedulingSecs": 15,
+        "runningSecs": 108,
+        "totalSecs": 124
+      },
+      "status": {
+        "stage": "ERROR",
+        "message": "Job failed with exit code: 1. Reason: Error",
+        "failureCount": 0
+      },
+      "conservative_billable_minutes": 3,
+      "pricing_unit_usd": 0.041667,
+      "estimated_upper_bound_usd": 0.125001
+    },
+    {
+      "id": "6abd00604c46ef1987036497",
+      "flavor": "a100-large",
+      "startedAt": "2026-09-30T12:28:24.196Z",
+      "finishedAt": "2026-09-30T12:31:55.114Z",
+      "durations": {
+        "schedulingSecs": 7,
+        "runningSecs": 210,
+        "totalSecs": 218
+      },
+      "status": {
+        "stage": "COMPLETED",
+        "message": null,
+        "failureCount": 0
+      },
+      "conservative_billable_minutes": 4,
+      "pricing_unit_usd": 0.041667,
+      "estimated_upper_bound_usd": 0.166668
+    },
+    {
+      "id": "6abd028f031314b696344aed",
+      "flavor": "h200",
+      "startedAt": "2026-09-30T12:37:43.301Z",
+      "finishedAt": "2026-09-30T12:41:20.011Z",
+      "durations": {
+        "schedulingSecs": 7,
+        "runningSecs": 216,
+        "totalSecs": 224
+      },
+      "status": {
+        "stage": "COMPLETED",
+        "message": null,
+        "failureCount": 0
+      },
+      "conservative_billable_minutes": 4,
+      "pricing_unit_usd": 0.083333,
+      "estimated_upper_bound_usd": 0.333332
+    },
+    {
+      "id": "6abd053cfbc85ba68235adf3",
+      "flavor": "h200",
+      "startedAt": "2026-09-30T12:49:08.277Z",
+      "finishedAt": "2026-09-30T13:00:31.350Z",
+      "durations": {
+        "schedulingSecs": 7,
+        "runningSecs": 683,
+        "totalSecs": 690
+      },
+      "status": {
+        "stage": "ERROR",
+        "message": "Job failed with exit code: 1. Reason: Error",
+        "failureCount": 0
+      },
+      "conservative_billable_minutes": 12,
+      "pricing_unit_usd": 0.083333,
+      "estimated_upper_bound_usd": 0.9999960000000001
+    }
+  ],
+  "next_job_hardware": "a100-large",
+  "next_job_rate_per_hour_usd": 2.50002,
+  "next_job_timeout_minutes": 165,
+  "conservative_combined_upper_bound_usd": 9.375055,
+  "final_job": {
+    "id": "6abd0d31404719ba37613271",
+    "flavor": "a100-large",
+    "startedAt": "2026-09-30T13:23:03.853Z",
+    "finishedAt": "2026-09-30T16:00:04.270Z",
+    "durations": {
+      "schedulingSecs": 6,
+      "runningSecs": 9420,
+      "totalSecs": 9426
+    },
+    "status": {
+      "stage": "COMPLETED",
+      "message": null,
+      "failureCount": 0
+    }
+  },
+  "final_job_estimated_upper_bound_usd": 6.5834,
+  "all_jobs_estimated_upper_bound_usd": 8.7919,
+  "completed_under_budget": true,
+  "successful_run_breakdown": {
+    "applies_to": "Original 60000-decision checkpoint, not separate 6000-decision ML Intern replay",
+    "job_id": "6abd0d31404719ba37613271",
+    "training_rows": 60000,
+    "optimizer_steps": 3720,
+    "training_phase_seconds": 7786.5,
+    "training_phase_minutes": 129.775,
+    "published_a100_hourly_rate_usd": 2.5,
+    "training_phase_estimate_usd": 5.4073,
+    "full_successful_job_conservative_estimate_usd": 6.5834,
+    "guide_successful_run_approximate_compute_usd": 6.6,
+    "earlier_failed_attempts_and_pilots_estimate_usd": 2.208502,
+    "historical_development_total_estimate_usd": 8.7919,
+    "pricing_source": "https://huggingface.co/docs/hub/jobs-pricing",
+    "pricing_verified_date": "2026-10-01",
+    "training_duration_source": "train_done event in the successful job logs; 7786.5 seconds, 3720 steps, 60000 rows",
+    "invoice_verified": false,
+    "excluded": [
+      "HuggingChat inference",
+      "local CPU work",
+      "operator time",
+      "separate later 6k replay"
+    ],
+    "guide_note": "Runtime-based estimate for one successful run; not a fixed price guarantee."
+  }
+}

JOB-LEDGER.json

@@ -0,0 +1,106 @@
+{
+  "checked_at": "2026-09-30T16:07:03.089274+00:00",
+  "authorized_total_budget_usd": 10,
+  "prior_jobs_conservative_reserve_usd": 2.5,
+  "current_job_hardware_ceiling_usd": 6.875,
+  "combined_upper_bound_usd": 9.375,
+  "invoice_verified": false,
+  "billing_account": "OpenMed",
+  "chat_tokens_billing_account": "MaziyarPanahi; accepted separately by user",
+  "full_60000_training_started": true,
+  "jobs": [
+    {
+      "id": "6abce8d0031314b6963447a4",
+      "hardware": "cpu-basic",
+      "status": "ERROR",
+      "phase": "credential probe",
+      "optimizer_steps": 0,
+      "failure": "Connector lacked private repository write/bucket mount permission; user authorized existing local credential."
+    },
+    {
+      "id": "6abcf03c031314b69634487d",
+      "hardware": "a100-large",
+      "status": "ERROR",
+      "phase": "first pilot",
+      "optimizer_steps": 0,
+      "failure": "Datasets5 lazy Column used as NumPy array; bulk materialization fixed"
+    },
+    {
+      "id": "6abcf3be4c46ef198703619a",
+      "hardware": "a100-large",
+      "status": "ERROR",
+      "phase": "repaired pilot before prototype",
+      "optimizer_steps": 25,
+      "failure": "Reload assertion compared Trainer BF16-wrapped output against plain FP32 output; saved checkpoints retained",
+      "checkpoint_retained": true
+    },
+    {
+      "id": "6abcfa794c46ef1987036338",
+      "hardware": "a100-large",
+      "status": "ERROR",
+      "phase": "FlashAttention dependency check",
+      "optimizer_steps": 0,
+      "failure": "kernels0.17.1 outside Transformers5.17 allowed range [0.16.0,0.17.0)"
+    },
+    {
+      "id": "6abcff404c46ef1987036446",
+      "hardware": "a100-large",
+      "status": "ERROR",
+      "phase": "CUDA kernel resolution check",
+      "optimizer_steps": 0,
+      "failure": "Pinned model-repository SHA was looked up in kernel-repository type by kernels0.16; repositories have distinct SHAs"
+    },
+    {
+      "id": "6abd00604c46ef1987036497",
+      "hardware": "a100-large",
+      "status": "COMPLETED",
+      "phase": "corrected kernel check then measured pilot and conditional prototype",
+      "timeout_seconds": 7200,
+      "hardware_ceiling_usd": 5.0,
+      "optimizer_steps": 25,
+      "kernel_forward_backward_verified": true,
+      "checkpoint_reload_verified": true,
+      "full_prototype_started": false,
+      "stop_reason": "Measured unbucketed A100 throughput did not fit the two-hour deadline."
+    },
+    {
+      "id": "6abd028f031314b696344aed",
+      "hardware": "h200",
+      "status": "COMPLETED",
+      "phase": "matching-batch pilot then conditional 60,000-decision prototype",
+      "timeout_seconds": 4800,
+      "hardware_ceiling_usd": 6.66664,
+      "optimizer_steps": 40,
+      "checkpoint_reload_verified": true,
+      "full_prototype_started": false,
+      "measured_rows_per_second": 18.72,
+      "stop_reason": "Pilot estimate plus conservative setup/evaluation reservations exceeded the 80-minute wrapper allowance."
+    },
+    {
+      "id": "6abd053cfbc85ba68235adf3",
+      "hardware": "h200",
+      "status": "ERROR",
+      "phase": "full 60,000-decision prototype using frozen successful H200 pilot",
+      "timeout_seconds": 5340,
+      "hardware_ceiling_usd": 7.41664,
+      "pilot_results_revision": "ef6c274e502adb27814f80e492ff5904c4116bfd",
+      "optimizer_steps": 0,
+      "failure": "TrainingArguments5.17 removed warmup_ratio; prototype-only branch was not included in pilot"
+    },
+    {
+      "id": "6abd0d31404719ba37613271",
+      "hardware": "a100-large",
+      "status": "COMPLETED",
+      "phase": "repaired actual prototype; lazy batch tokenization; warmup_steps0.03",
+      "timeout_seconds": 9900,
+      "hardware_ceiling_usd": 6.875,
+      "recipe_revision": "b402b938f096ec3bd1bcc25c5655b6c4b16e64e1",
+      "optimizer_steps_verified": 3720,
+      "rows_seen_verified": 60000,
+      "first_optimizer_seconds_after_train_process_start": 44.8,
+      "finished_at": "2026-09-30T16:00:04.270Z",
+      "estimated_compute_upper_bound_usd": 6.5834
+    }
+  ],
+  "all_jobs_estimated_cost_usd": 8.7919
+}

PUBLIC-PROMPT-SHORT.md

@@ -0,0 +1,19 @@
+# Short HuggingChat ML Intern prompt
+
+Before sending: select the account/org that should pay for chat-model tokens. Set the same intended namespace for GPU Jobs.
+
+```text
+Help me train my own small Jev-style decision model with HuggingChat ML Intern.
+
+Use answerdotai/ModernBERT-base and MaziyarPanahi/AgentToolDecisions-180K. Train on 60,000 frozen training decisions from next-action and tool-selection, with 4,096-token input pairs. Rank each decision's declared choices using a shared encoder and scalar scoring head.
+
+My Job billing namespace: <YOUR_ACCOUNT_OR_ORG>
+New PRIVATE model repo: <YOUR_ACCOUNT_OR_ORG>/ModernJEV-Decide-Preview
+Total compute budget, including failed jobs: $10.
+
+First show me a read-only plan and budget; wait for my authorization before launching anything. After approval, validate dependencies and CUDA, measure a short A100 pilot, and proceed only if the full prototype fits the budget. Keep the official splits. Evaluate all 1,700 in-scope test decisions and all 3,652 unseen-task When2Call test rows separately, without training or tuning on When2Call. Report per-task scores against constant-majority and allowed-choice frequency baselines; never report an overall accuracy. Test arbitrary answer labels and lists, including candidate order and label renaming. Save weights before evaluation, plus a useful README, tested prediction helper, actual coverage, costs, and the reproducible recipe.
+
+Use JSONL metrics. No Spaces, no visibility changes, no automatic retries or budget extensions. Keep the model private until I authorize release. Do not claim Jev parity.
+```
+
+For implementation details and lessons from this run, use PUBLIC-PROMPT.md. This short prompt is a starting request; review the generated executable plan before granting compute.

PUBLIC-PROMPT.md

@@ -0,0 +1,137 @@
+# Train your own small agent decision model with HuggingChat
+
+Open https://huggingface.co/chat/, enable **ML Intern**, replace the three settings below, and paste the prompt. Start without granting compute. After reviewing the proposal, grant the stated budget to authorize the pilot and bounded prototype. Builders need their own Hugging Face compute credits and permission to write the destination model repository.
+
+**Check both billing accounts:** select the intended user or organization inside HuggingChat before using the coding model, and pass that namespace on every Hugging Face Job. A Job's organization does not automatically change the account charged for chat-model tokens.
+
+```text
+NAMESPACE = <your HF username or organization>
+MODEL_REPO = <NAMESPACE>/ModernJEV-Decide-Preview
+TRAINING_DECISIONS_TARGET = 60000
+MAX_SEQUENCE_LENGTH = 4096
+TOTAL_COMPUTE_BUDGET_USD = 10
+
+Help me train a small open decision model from
+MaziyarPanahi/AgentToolDecisions-180K using HuggingChat ML Intern.
+Do a read-only preflight first, show your executable plan and budget,
+then STOP for my explicit authorization before any compute job,
+sandbox, repository creation, training or Hub write.
+Use read-only Hub metadata and dataset previews for that preflight.
+Any executable dataset audit, model-load or GPU check belongs in the
+approved pilot; do not provision compute to satisfy a zero-spend step.
+
+The model should read an agent's state, a question, and the allowed
+choices, then return one of those choices. It is a Jev-style choice
+scorer, not a reproduction of Jev and not a chatbot.
+
+DATA
+Pin dataset revision f2fb14e4ec977c420f376c08785664cd38763d7e.
+Keep the published train/validation/test splits; never resplit.
+Train ONLY agent_next_action_type and tool_selection.
+Their combined train/validation/test counts are 112973/1798/1700.
+Inspect actual rows, label balance and input lengths before training.
+Exclude the single-label tool_or_text_action,
+tool_argument_completeness and tool_response_preference families,
+and exclude the test-only when_to_call_tool family from training.
+After freezing the final checkpoint, evaluate all 3,652 When2Call
+test rows separately as unseen-task transfer, without tuning on them.
+Report only per-task scores, never overall or macro accuracy.
+Reverify constant-majority references: 616/1158 next-action,
+79/542 tool-selection, and 1295/3652 When2Call. Distinguish these
+from the stronger allowed-choice training-frequency baseline.
+When2Call majority is a descriptive test-set reference, not fitted.
+Next-action has three declared choices but only two gold labels;
+tool_and_response is never gold. Do not imply three-class coverage.
+Assert no group_id crosses splits. For source-row leakage, use the
+full tuple (source_dataset, source_revision, source_config,
+source_split, source_row_id), never bare source_row_id.
+
+MODEL AND LOSS
+Start with answerdotai/ModernBERT-base at revision
+8949b909ec900327062f0ebf497f51aef5e6f0c8.
+Use AutoModelForSequenceClassification(num_labels=1,
+attn_implementation='sdpa'). Score each (question + state,
+candidate label + criterion) pair. Apply softmax cross-entropy
+within each decision's row_id; group_id is an episode, NOT the
+loss grouping key. Score only the row's declared choices.
+Support arbitrary caller-supplied labels and unique answer-string lists;
+test 2/3/7/20 choices, reordered candidates and renamed labels.
+Report API acceptance separately from semantic correctness.
+Do not feed gold labels, gold JSON, gold scores or provenance into
+model inputs. Do not include candidate indices in candidate text.
+Verify trainer and dependency APIs against installed versions.
+In Transformers 5.17, use warmup_steps=0.03 for 3% warmup;
+warmup_ratio was removed. Instantiate the complete production
+TrainingArguments branch before any expensive data preparation.
+Test the full trainer/validation/checkpoint path locally on a tiny
+subset, not only the separate pilot branch.
+Prefer lazy batch tokenization with prefetching so optimizer steps
+begin without an upfront full-dataset mapping pass.
+Do not substitute a different ranking objective without approval.
+For long 4096-token inputs, benchmark ModernBERT's documented
+FlashAttention backend as well as SDPA. Match the installed Torch/CUDA
+ABI to an available prebuilt kernel; do not assume the newest Torch
+release has a matching kernel. Pin and record the kernel revision. For the verified environment in this
+run: torch 2.12.0 + cu126, transformers 5.17.0, kernels 0.16.0;
+Transformers 5.17 rejects kernels 0.17.x. The pinned FlashAttention
+revision is f50dc99ed079b35990bc895d43fd353ea0cb376d.
+Inspect the kernel repository type used by the installed loader. The
+model repository and kernel repository may have different revisions.
+Before data loading, run a small CUDA forward/backward kernel smoke.
+Datasets 5 columns are lazy: bulk-materialize metadata and NumPy arrays
+before numerical comparisons or sorting. Tokenize only sampled training
+candidates if using a sampled pool; evaluation must rank all candidates.
+Verify checkpoint parameter equality and compare predictions at the
+same precision. Do not compare a Trainer BF16-wrapped forward to FP32
+inference with an unrealistically strict tolerance.
+
+INPUTS
+The audited 192-row sample had 2221 candidate pairs: full-input
+p50 2460 tokens, p95 3935, max 5179. At 2048, 1643 pairs truncate.
+Start the pilot at 4096 tokens. Preserve question and candidate;
+document any state truncation and measure its frequency. Remove
+duplicated policy only when exactly equal to the first system
+message. Batch by candidate count/token cost and measure GPU
+memory. Never claim all input was read when it was truncated.
+
+RUN
+Pass NAMESPACE on EVERY job so billing uses the intended account.
+Check current hardware price and private Hub write permissions.
+If write access is missing, ask me to reconnect with the appropriate
+repository permissions; never request a token pasted into the chat.
+Use one A100 80GB (a100-large) if available at $2.50/hour:
+first a 25-step pilot with a 20-minute timeout, then at most one
+prototype job with a 165-minute timeout. Set a total $10 compute budget.
+Select exactly TRAINING_DECISIONS_TARGET rows from the training split,
+stratified by task family with seed 42; freeze and hash their row IDs.
+Use measured pilot throughput to check whether one complete epoch fits,
+leaving time for validation, final evaluation and upload.
+Record the actual rows, steps, elapsed time and cost. No guarantee
+of a full epoch, no parallel jobs, no automatic retries or extensions.
+If the pilot makes the deadline infeasible, stop and report it.
+
+Create a NEW PRIVATE model repository only after authorization.
+Save checkpoints, tokenizer, input template, prediction code,
+dependency pins, metrics and a model card there. Do not overwrite
+an existing repository. Persist the pilot before its job exits.
+Never create a Hugging Face Space or change any Space visibility.
+Use local JSONL metrics; optionally use Trackio only after verifying
+space_id=None performs local logging without creating a Space.
+Never call create_trackio or provision a dashboard Space.
+Never paste tokens into chat, code, logs or artifacts; use secrets.
+
+EVALUATION AND DELIVERY
+Use a fixed one-epoch recipe and its final checkpoint for the first
+prototype; monitor validation without selecting on the test set.
+If the time limit stops training early, disclose the actual row coverage.
+Evaluate the final checkpoint
+once on the 1700 in-scope official test decisions, per family.
+Compare with uniform-choice and training-derived majority/frequency
+baselines restricted to the declared candidates. Check shuffled
+candidate order and report label agreement, with a stated tie rule.
+Return typed-choice prediction code and a reproducible recipe.
+Preserve source attribution and inspect licenses before release.
+Report failures and unmeasured results honestly. Do not claim Jev
+parity, general agent competence or support for other primitives.
+Keep weights private until I separately authorize publication.
+```

README.md

@@ -0,0 +1,230 @@
+---
+license: apache-2.0
+library_name: transformers
+pipeline_tag: text-classification
+base_model: answerdotai/ModernBERT-base
+base_model_relation: finetune
+datasets:
+  - MaziyarPanahi/AgentToolDecisions-180K
+tags:
+  - modernbert
+  - encoder
+  - decision-model
+  - tool-routing
+  - agentic
+  - preview
+language:
+  - en
+---
+
+<div align="center">
+
+# ModernJEV-Decide-Preview
+
+### Give your agent a next move.
+
+**A small encoder for choosing an action or a tool from the options you provide.** We trained this 60,000-decision prototype for **about $6.60 in A100 compute, including setup and evaluation**. The training phase itself accounted for **about $5.40**. Both figures are estimates from the measured runtime.
+
+149.6M parameters · 4,096-token inputs · Typed choices · Built on ModernBERT
+
+</div>
+
+> **Experimental preview.** The final checkpoint processed **60,000/60,000 selected training decisions**. Held-out results cover **1,158 next-action**, **542 tool-selection**, and **3,652 unseen When2Call** decisions, reported separately. This is a small choice-ranking prototype, not a Jev reproduction.
+
+An agent doesn't need to write a paragraph every time it makes a decision. Sometimes the useful output is simply **answer**, **call a tool**, or **choose this tool**.
+
+ModernJEV-Decide-Preview reads a conversation, its policy and available tools, then scores the choices supplied with your question. Your application receives one of those labels, along with the scores for the alternatives. The model ranks choices; your application executes the selected action.
+
+The architecture is a **ModernBERT cross-encoder with one scalar scoring head**. Choice labels and descriptions are input text, so the output is not limited to a fixed vocabulary of tool names. This is a Jev-style choice model built from public agent decisions, with its own training and evaluation; it is not a reproduction of Jev.
+
+![How decisions are scored](architecture.svg)
+
+## Where it fits
+
+| Use case | Supply | Receive |
+|---|---|---|
+| **Next-action routing** | Current conversation, policy, tool list | A declared action such as `text_response` or `tool_call` |
+| **Tool selection** | The task and each available tool's purpose | The label of the selected tool |
+| **Workflow branching** | A text state and clearly described alternatives | One branch label; evaluate on your own workflow before adoption |
+| **Decision-model experiments** | Your choices and held-out tasks | Candidate rankings you can inspect and compare |
+
+Useful places to start: support agents choosing between lookup and escalation, assistants selecting an API, and workflows choosing between a direct answer and a tool-backed step. The model does not generate tool arguments or execute tools.
+
+## Train a decision model with HuggingChat ML Intern
+
+We tested this workflow with **one execution message, an attached tested recipe, and a budget setup click**. In a fresh HuggingChat conversation, ML Intern prepared the code, ran its own checks, launched an OpenMed A100 job, trained **6,000 decisions in 1,500 optimizer steps**, and uploaded a new private checkpoint. No follow-up implementation prompt, Codex code repair or Codex job launch was needed in that conversation. Training took **996.9 seconds (16 minutes 37 seconds)**; that excludes chat preparation, setup and subsequent evaluation.
+
+**These are two separate runs:**
+
+| | Weights in this repository | ML Intern recipe replay |
+|---|---|---|
+| Selected training decisions | 60,000 | 6,000 |
+| Optimizer steps | 3,720 | 1,500 |
+| Execution | Started with ML Intern; Codex debugged, launched and completed the run | ML Intern executed from one message with supplied source files |
+| Evidence | Per-task model results below | Exact row coverage and checkpoint upload independently verified |
+| Status at this documentation revision | Training and evaluation complete | Training and fresh checkpoint verified; operator stopped after workflow proof |
+
+The first replay attempt stopped **before GPU spend** because HuggingChat's tools could not read the existing private source repository, although the local HF login could. We preserved that failed attempt and opened a fresh conversation with the source files attached. The successful training therefore demonstrates **one-prompt execution of a supplied recipe after setup**, not first-attempt success, code invention from scratch, or a guarantee for arbitrary training tasks. ML Intern made its own pre-launch code corrections. After the first GPU run had trained and verified checkpoint reload, its finalizer failed because stdin execution did not define `__file__`. It detected that failure and a reversed When2Call baseline unpack itself, then used the authorized corrective retry. The retry also trained all 6,000 rows and saved a checkpoint. The operator then stopped the remaining evaluations and CPU sandbox because the requested workflow proof was already established. Full replay packaging was not completed. These autonomous repairs, failed compute and the operator termination are part of the recorded run.
+
+### Try the same workflow
+
+1. Open [HuggingChat](https://huggingface.co/chat/) and enable **ML Intern**. Our replay used **GLM-5.3-Flash**.
+2. Select your intended billing account or organization in HuggingChat settings. Ensure you can create/write a private model repository and submit Jobs in that namespace.
+3. Download [the execution prompt](workflow/PROMPT-R2.txt) and [the tested source bundle](workflow/recipe-bundle.txt). Change the billing namespace and destination to your own **new private model repository** throughout the prompt. Keep the pinned source revisions. Do not reuse our destination name.
+4. Attach both files to one message and send the instruction below. Set the conversation's compute-budget control to **$10**; the written budget alone does not replace that UI setting.
+5. Let ML Intern do its checks, execution and monitoring. A submitted job is not proof of a trained model: check row coverage, saved weights, reload, costs and per-task evaluations.
+
+This was the actual execution message accompanying our attachments:
+
+```text
+Execute PROMPT-R2.txt now in ML Intern using the supplied recipe-bundle.txt.
+OpenMed is the billing namespace. This single message authorizes the bounded
+experiment described in the prompt. The earlier attempt stopped before compute
+because the private source lookup failed; use these supplied source files,
+and preserve that fact in the final report.
+```
+
+For your run, replace `OpenMed` in that message with your billing namespace as well. The full attached prompt is part of the instruction, not optional background. It specifies the 6,000-row selection, 4,096-token inputs, rehearsal, privacy, one A100, timeout, total compute cap, at most one bounded retry, and per-task evaluation requirements. Do not claim our measured runtime or results for a new run before measuring them.
+
+[Exact prompt](workflow/PROMPT-R2.txt) · [Source bundle and hashes](workflow/recipe-manifest.json) · [Run provenance](workflow/ONE-PROMPT-REPLAY.md) · [Replay job](https://huggingface.co/jobs/OpenMed/6abe2b6ffbc85ba6823612cc)
+
+## Quick start
+
+Install PyTorch and Transformers, then use the included inference helper so the input formatting matches training:
+
+```bash
+pip install "torch==2.12.0" "transformers==5.17.0" "huggingface-hub==1.33.0"
+```
+
+Download the supplied helper, then run it alongside your application. During the private preview, your Hugging Face account must have access to the repository.
+
+```bash
+hf download MaziyarPanahi/ModernJEV-Decide-Preview predict.py --local-dir modernjev
+cd modernjev
+```
+
+The included helper was executed against this checkpoint. See [the observed example output](example-output.json); the snippet below prints the actual prediction rather than promising a fixed answer.
+
+```python
+from predict import DecisionModel
+
+model = DecisionModel("MaziyarPanahi/ModernJEV-Decide-Preview")
+decision = model.decide(
+    state={
+        "policy": "Use the order lookup tool when a customer asks about an order.",
+        "conversation": [
+            {"role": "user", "content": "Where is order A123?"}
+        ],
+        "available_tools": [
+            {"name": "lookup_order", "description": "Retrieve an order's delivery status."},
+            {"name": "search_catalog", "description": "Find products in the catalog."},
+        ],
+    },
+    question="Which tool should the assistant call next?",
+    criteria={
+        "lookup_order": "Retrieve the delivery status of the customer's order.",
+        "search_catalog": "Search for products that match a shopping request.",
+    },
+)
+print(decision["predicted_label"])
+print(decision["candidates"])
+```
+
+You can also supply `criteria=["Escalate to a human", "Search the product catalog"]` when each answer is its own description. The result contains the selected label, all candidate scores, and whether input truncation occurred. Scores are normalized **within the supplied choice set**; they are not calibrated confidence estimates and cannot be compared directly across unrelated requests.
+
+## Training recipe
+
+| Setting | Recorded configuration |
+|---|---|
+| Base | `answerdotai/ModernBERT-base` |
+| Base revision | `8949b909ec900327062f0ebf497f51aef5e6f0c8` |
+| Parameters with scalar head | 149,605,633 |
+| Input limit | 4,096 tokens including both sequences and special tokens |
+| Dataset | `MaziyarPanahi/AgentToolDecisions-180K` |
+| Dataset revision | `f2fb14e4ec977c420f376c08785664cd38763d7e` |
+| Selected training target | Exactly 60,000 train-split decisions, stratified by family, seed 42 |
+| Actual training coverage | 60,000 decisions; 3,720 optimizer steps |
+| Attention backend | kernels-community/flash-attn2@f50dc99ed079b35990bc895d43fd353ea0cb376d |
+| Training elapsed | 129.8 minutes |
+| GPU | NVIDIA A100-SXM4-80GB |
+| Task families | `agent_next_action_type`, `tool_selection` |
+| Objective | Per-decision softmax cross-entropy over candidate scores |
+| Candidate grouping | `row_id`, preserving `group_id` only for episode/split boundaries |
+| Candidate pool | Gold plus up to three declared negatives per training decision; evaluation ranks every declared choice |
+| Evaluation | Monitor official validation; evaluate the final fixed-epoch checkpoint on 1,700 in-scope official test decisions |
+
+The dataset has 180,000 total rows. This prototype targets 60,000 of its 112,973 in-scope training decisions; **180,000 is not the number used for training**. Exact selected row IDs, coverage, steps and dependency versions accompany this checkpoint. This repository contains model artifacts; it does not expose an inference endpoint.
+
+## Evaluation
+
+Every task is reported separately. The checkpoint was fixed after one pass over the selected training rows; no When2Call labels were used for training, checkpoint selection or hyperparameter tuning. The three single-label task families are excluded from quality claims.
+
+| Task | Correct / cases | Model accuracy | Constant-majority reference | Lift |
+|---|---:|---:|---:|---:|
+| Next action | 850 / 1,158 | 73.40% | 53.20% | +20.21 pp |
+| Tool selection | 337 / 542 | 62.18% | 14.58% | +47.60 pp |
+| When2Call — unseen task family | 1,256 / 3,652 | 34.39% | 35.46% | -1.07 pp |
+
+**Transfer limitation:** When2Call is below its majority reference. Accepting a new question and new answer labels does not establish that the model understands that task reliably. This preview demonstrates gains on its trained task families; general decision-making and clinical use remain unvalidated.
+
+The next-action majority label is `text_response`; the tool-selection majority label is `transfer_to_human_agent`. Both are also the most frequent labels in the selected training subset. The fixed tool label is absent from 235 of the 542 choice lists, so it is a descriptive constant-label reference, not a valid per-row router. When2Call has no training rows: its reference is the test-set majority (a tie between `cannot_answer` and `tool_call`), not a fitted training baseline.
+
+The stronger **allowed-choice training-frequency baseline** always chooses among the supplied answers. It is distinct from the constant-majority reference above.
+
+| Comparison | Next action (1,158 cases) | Tool selection (542 cases) |
+|---|---:|---:|
+| ModernJEV-Decide-Preview | 73.40% | 62.18% |
+| ModernBERT + untrained scalar head, seed 42 | 42.83% | 0.74% |
+| Allowed-choice training-frequency baseline | 53.20% | 22.88% |
+| Uniform over supplied choices, expected accuracy | 33.33% | 5.19% |
+
+Next-action rows have three choices. Tool-selection rows have a median of 20 choices (range 11–32). When2Call rows have four choices; uniform expected accuracy is 25%. Evaluation ranks all declared candidates, including choices not sampled during training.
+
+Untrained ModernBERT is a pretrained encoder with a randomly initialized scalar head, not a trained decision classifier. No frozen-backbone trained-head comparison or Jev API comparison was completed in this bounded run.
+
+### Open answer labels
+
+The helper accepts arbitrary string labels with descriptions, or a list of unique answer strings. It has no fixed output-class vocabulary. Final-checkpoint interface checks passed with 2, 3, 7 and 20 supplied answers, including reversed order and fresh opaque labels. These are interface checks, not a broad capability benchmark.
+
+For one illustrative routing scenario at four answer-list sizes, the intended description was selected at 2, 3 and 7 choices, but not at 20. Renaming labels while retaining descriptions changed the selected description in the 3- and 7-choice variants. **Label wording matters:** use descriptive labels and validate your own question/answer sets. The small illustrative checks are published in full, including the misses.
+
+Candidate-order label agreement on the official test subset was **100% across five permutations of 299 decisions**. Reordering a fixed set of label strings is a different test from renaming those strings. Ties use lexical label order.
+
+GPU latency for **one candidate forward pass**, excluding tokenization: median **23.6 ms**, p95 **24.2 ms**. A full decision scores multiple candidates; these are not end-to-end decision timings. The two trained tasks were evaluated with GPU BF16 inference. Supplemental When2Call and open-label checks used local CPU FP32 inference on the same saved weights.
+
+See [per-task results](per-task-results.json), [all When2Call predictions](evaluation/when2call-predictions.jsonl), [open-label checks](evaluation/open-label-checks.json), [baseline definitions](evaluation/per-task-baselines.json), [training coverage](training_coverage.json), [runtime versions](runtime-versions.json) and [the pinned training recipe](recipe/train.py). The split audit found no cross-split episode or full upstream source conflicts. The 4,096-token limit truncated 414 candidate pairs in the two trained task tests; no When2Call rows were truncated.
+
+### Compute accounting
+
+**For a successful run of the 60k recipe, allow about $6.60 in GPU compute at our measured runtime.** We used one A100 80GB at the published [Hugging Face Jobs rate of $2.50/hour](https://huggingface.co/docs/hub/jobs-pricing).
+
+| Scope | Measured time | Estimated GPU compute |
+|---|---:|---:|
+| Training phase: 60,000 decisions, 3,720 optimizer steps | 129.8 minutes | **$5.41** |
+| Full successful job: setup, training, evaluation and packaging | 157 minutes | **$6.58**, conservatively rounded; about **$6.60** |
+| Earlier development attempts and pilots | Across multiple jobs | **$2.21** |
+| Original development total, including the successful job | Across all original jobs | **$8.79** |
+
+The **$8.79** figure includes our earlier failed attempts and pilots; it is not the cost of the final successful run. The $10 authorization was a spending cap, not a quoted training price. These figures refer to the original **60k checkpoint in this repository**, separately from the later 6k ML Intern workflow test.
+
+Costs are runtime/rate estimates, not reconciled invoice amounts. They exclude HuggingChat inference, local CPU work and operator time; another run's runtime may differ. See [cost accounting](COST-AUDIT.json) for the job-level receipts and training-phase calculation.
+
+## Input design matters
+
+- Give every choice a concrete, distinct description. Include the same available tool information the application actually has.
+- Use the supplied serialization helper. Long state may need truncation at the 4,096-token limit; the helper reports it.
+- The training data is English agent conversations and tool decisions. New domains and arbitrary workflow labels need their own evaluation.
+- `tool_and_response` is a declared next-action choice but never a gold label in this dataset. That action is outside demonstrated positive training coverage.
+- This preview implements **choice ranking**. It does not implement Jev's other primitives, vision inputs, generated explanations, or autonomous tool execution.
+
+## Reproduce the 60k model or the 6k workflow
+
+Use [the original 60k training recipe](recipe/train.py) and [its detailed training prompt](PUBLIC-PROMPT.md) to inspect how these weights were built. Use the **6k ML Intern replay instructions above** to test the one-message execution workflow. Their scope, checkpoints, runtimes and metrics are different. All model-quality results on this page belong to the original 60k checkpoint.
+
+## Attribution
+
+Base model: [Answer.AI ModernBERT](https://huggingface.co/answerdotai/ModernBERT-base), Apache 2.0.
+
+Decision dataset: [MaziyarPanahi/AgentToolDecisions-180K](https://huggingface.co/datasets/MaziyarPanahi/AgentToolDecisions-180K), transformed from the upstream agentic sources documented in its card. Keep the pinned revisions and source attribution with derivative work, and follow the upstream dataset licenses when redistributing data.
+
+**Original 60k workflow:** HuggingChat ML Intern prepared the proposal, training code, helper and model-card draft. Codex audited the data and corrected candidate targets, padding budgets, coverage accounting and checkpoint verification. Hugging Face Jobs was launched with the operator's approved local HF credential after the connector's write permissions returned 403. This is an assisted training workflow; HuggingChat did not autonomously execute the training job.

config.json

@@ -0,0 +1,84 @@
+{
+  "architectures": [
+    "ModernBertForSequenceClassification"
+  ],
+  "attention_bias": false,
+  "attention_dropout": 0.0,
+  "bos_token_id": 50281,
+  "classifier_activation": "gelu",
+  "classifier_bias": false,
+  "classifier_dropout": 0.0,
+  "classifier_pooling": "mean",
+  "cls_token_id": 50281,
+  "decoder_bias": true,
+  "deterministic_flash_attn": false,
+  "dtype": "float32",
+  "embedding_dropout": 0.0,
+  "eos_token_id": 50282,
+  "global_attn_every_n_layers": 3,
+  "gradient_checkpointing": false,
+  "hidden_activation": "gelu",
+  "hidden_size": 768,
+  "id2label": {
+    "0": "LABEL_0"
+  },
+  "initializer_cutoff_factor": 2.0,
+  "initializer_range": 0.02,
+  "intermediate_size": 1152,
+  "label2id": {
+    "LABEL_0": 0
+  },
+  "layer_norm_eps": 1e-05,
+  "layer_types": [
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention",
+    "sliding_attention",
+    "sliding_attention",
+    "full_attention"
+  ],
+  "local_attention": 128,
+  "max_position_embeddings": 8192,
+  "mlp_bias": false,
+  "mlp_dropout": 0.0,
+  "model_type": "modernbert",
+  "norm_bias": false,
+  "norm_eps": 1e-05,
+  "num_attention_heads": 12,
+  "num_hidden_layers": 22,
+  "pad_token_id": 50283,
+  "position_embedding_type": "absolute",
+  "rope_parameters": {
+    "full_attention": {
+      "rope_theta": 160000.0,
+      "rope_type": "default"
+    },
+    "sliding_attention": {
+      "rope_theta": 10000.0,
+      "rope_type": "default"
+    }
+  },
+  "sep_token_id": 50282,
+  "sparse_pred_ignore_index": -100,
+  "sparse_prediction": false,
+  "tie_word_embeddings": true,
+  "transformers_version": "5.17.0",
+  "use_cache": false,
+  "vocab_size": 50368
+}

evaluation/evaluate_open_labels.py

@@ -0,0 +1,169 @@
+"""Supplemental frozen-checkpoint evaluation; no training or Hub writes.
+
+Run --audit-only first. After final checkpoint is saved, supply --model and,
+for a Hub model, --revision. CPU is default; JSONL predictions resume safely.
+"""
+import argparse
+import hashlib
+import importlib.util
+import json
+import statistics
+from collections import Counter
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent
+DATA_REV = 'f2fb14e4ec977c420f376c08785664cd38763d7e'
+CACHE = Path('/private/tmp/atd-ml-intern-hub-cache/MaziyarPanahi___agent_tool_decisions-180_k/default/0.0.0') / DATA_REV
+FAMILIES = ('agent_next_action_type', 'tool_selection', 'when_to_call_tool')
+EXPECTED = {FAMILIES[0]: (1158, 616), FAMILIES[1]: (542, 79), FAMILIES[2]: (3652, 1295)}
+
+
+def write_json(path, value):
+    temp = path.with_suffix('.tmp')
+    temp.write_text(json.dumps(value, indent=2) + '\n')
+    temp.replace(path)
+
+
+def load_test():
+    from datasets import Dataset
+    return Dataset.from_file(str(CACHE / 'agent_tool_decisions-180_k-test.arrow'))
+
+
+def audit(test):
+    report = {'dataset_revision': DATA_REV, 'families': {}}
+    for family in FAMILIES:
+        rows = [r for r in test if r['task_family'] == family]
+        labels = Counter(r['gold_label'] for r in rows)
+        size, correct = EXPECTED[family]
+        assert len(rows) == size and max(labels.values()) == correct
+        candidates = [len(json.loads(r['criteria_json'])) for r in rows]
+        assert all(r['gold_label'] in json.loads(r['criteria_json']) for r in rows)
+        report['families'][family] = {
+            'n': size, 'majority_correct': correct, 'majority_accuracy': correct / size,
+            'majority_labels': sorted(k for k,v in labels.items() if v == correct),
+            'candidate_count': {'median': statistics.median(candidates),
+                                'min': min(candidates), 'max': max(candidates)},
+            'uniform_expected_accuracy': statistics.mean(1/n for n in candidates),
+            'reference_definition': 'Descriptive test-set constant-label majority; no model fitting',
+            'unseen_task_family': family == 'when_to_call_tool',
+        }
+    # Verify When2Call is absent from every training shard, not just the subset.
+    from datasets import Dataset
+    train_families = Counter()
+    selected_ids = set((ROOT / 'selected-row-ids.txt').read_text().splitlines())
+    train_gold = {f: Counter() for f in FAMILIES[:2]}
+    for path in sorted(CACHE.glob('*-train-*.arrow')):
+        columns = Dataset.from_file(str(path)).select_columns(['task_family','row_id','gold_label'])[:]
+        for family, row_id, gold in zip(columns['task_family'], columns['row_id'], columns['gold_label']):
+            train_families[family] += 1
+            if row_id in selected_ids and family in train_gold:
+                train_gold[family][gold] += 1
+    assert sum(train_families.values()) == 171056
+    assert train_families['when_to_call_tool'] == 0
+    assert sum(sum(c.values()) for c in train_gold.values()) == 60000
+    report['when2call_training_rows'] = 0
+    for family, counts in train_gold.items():
+        label = min(counts, key=lambda k: (-counts[k], k))
+        rows = [r for r in test if r['task_family'] == family]
+        report['families'][family]['training_fixed_majority'] = {
+            'label': label, 'correct': sum(r['gold_label'] == label for r in rows),
+            'n': len(rows),
+            'majority_label_absent_from_choices': sum(label not in json.loads(r['criteria_json']) for r in rows),
+        }
+    return report
+
+
+def open_labels(client):
+    checks = []
+    for n in (2, 3, 7, 20):
+        descriptions = ['Escalate this damaged parcel to a human support agent.',
+                        'Search the product catalog for a new item.']
+        descriptions += [f'Route to unrelated department number {i} for a different request.' for i in range(n-2)]
+        criteria = {f'fresh_choice_{n}_{i}': d for i,d in enumerate(descriptions)}
+        state = {'policy': 'Escalate damaged parcels to a human support agent.',
+                 'conversation': [{'role':'user','content':'My parcel arrived broken. I need help.'}]}
+        question = 'Which of these custom workflow branches should handle this request?'
+        outputs = []
+        variants = (criteria, dict(reversed(list(criteria.items()))),
+                    {f'opaque_{n}_{i}': d for i,d in enumerate(descriptions)}, descriptions)
+        for variant in variants:
+            answer = client.decide(state=state, question=question, criteria=variant, candidate_batch_size=4)
+            allowed = list(variant)
+            assert answer['predicted_label'] in allowed
+            assert sorted(c['label'] for c in answer['candidates']) == sorted(allowed)
+            assert len(answer['candidates']) == n
+            outputs.append(answer)
+        original_desc = criteria[outputs[0]['predicted_label']]
+        renamed_desc = variants[2][outputs[2]['predicted_label']]
+        checks.append({'answer_count': n, 'interface_pass': True,
+                       'reorder_same_label': outputs[0]['predicted_label'] == outputs[1]['predicted_label'],
+                       'rename_same_description': original_desc == renamed_desc,
+                       'selected_expected_description': original_desc == descriptions[0],
+                       'outputs': outputs})
+    return {'scalar_head': client.model.config.num_labels,
+            'interpretation': 'Interface checks and illustrative decisions, not a clinical benchmark or general accuracy estimate.',
+            'checks': checks}
+
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument('--audit-only', action='store_true')
+    p.add_argument('--interface-only', action='store_true')
+    p.add_argument('--model')
+    p.add_argument('--revision')
+    p.add_argument('--device', default='cpu')
+    p.add_argument('--output-dir', type=Path, default=ROOT / 'supplemental-evaluation')
+    args = p.parse_args()
+    out = args.output_dir; out.mkdir(parents=True, exist_ok=True)
+    test = load_test(); baseline = audit(test)
+    write_json(out/'per-task-baselines.json', baseline)
+    print(json.dumps(baseline), flush=True)
+    if args.audit_only: return
+    if not args.model: p.error('--model is required for checkpoint evaluation')
+    local = Path(args.model).is_dir()
+    if not local and not args.revision: p.error('Pin --revision for a Hub model')
+    if local:
+        weights = sorted(Path(args.model).glob('*.safetensors'))
+        assert weights, 'Local checkpoint must have saved safetensors'
+        h = hashlib.sha256()
+        for path in weights:
+            with path.open('rb') as f:
+                for chunk in iter(lambda:f.read(8*1024*1024), b''): h.update(chunk)
+        identity = h.hexdigest()
+    else: identity = args.revision
+    manifest = {'model': args.model, 'checkpoint_identity': identity, 'dataset_revision': DATA_REV}
+    if (out/'manifest.json').exists(): assert json.loads((out/'manifest.json').read_text()) == manifest
+    write_json(out/'manifest.json', manifest)
+    spec = importlib.util.spec_from_file_location('inference', ROOT/'predict_open_labels.py')
+    helper = importlib.util.module_from_spec(spec); spec.loader.exec_module(helper)
+    import torch
+    torch.set_num_threads(4)
+    client = helper.DecisionModel(args.model, device=args.device, revision=args.revision)
+    assert client.model.config.num_labels == 1
+    write_json(out/'open-label-checks.json', open_labels(client))
+    if args.interface_only: return
+    rows = [r for r in test if r['task_family'] == 'when_to_call_tool']
+    path = out/'when2call-predictions.jsonl'; previous = {}
+    if path.exists():
+        for line in path.read_text().splitlines():
+            if line: item = json.loads(line); previous[item['row_id']] = item
+    assert set(previous).issubset({r['row_id'] for r in rows})
+    with path.open('a') as f:
+        for row in rows:
+            if row['row_id'] in previous: continue
+            pred = client.decide(state=json.loads(row['state_json']), question=row['question_text'],
+                                 criteria=json.loads(row['criteria_json']), candidate_batch_size=4)
+            rec = {'row_id': row['row_id'], 'gold': row['gold_label'], **pred}
+            f.write(json.dumps(rec)+'\n'); f.flush(); previous[row['row_id']] = rec
+            if len(previous) % 100 == 0: print(f'When2Call {len(previous)}/{len(rows)}', flush=True)
+    correct = sum(previous[r['row_id']]['predicted_label'] == r['gold_label'] for r in rows)
+    n = len(rows); reference = baseline['families']['when_to_call_tool']['majority_accuracy']
+    result = {'task': 'when_to_call_tool', 'correct': correct, 'n': n, 'accuracy': correct/n,
+              'majority_reference': reference, 'lift_percentage_points': 100*(correct/n-reference),
+              'unseen_task_family': True, 'checkpoint_identity': identity,
+              'truncated_rows': sum(x['truncated'] for x in previous.values())}
+    write_json(out/'when2call-results.json', result)
+    print(json.dumps(result), flush=True)
+
+
+if __name__ == '__main__': main()

evaluation/open-label-checks.json

@@ -0,0 +1,1098 @@
+{
+  "scalar_head": 1,
+  "interpretation": "Interface checks and illustrative decisions, not a clinical benchmark or general accuracy estimate.",
+  "checks": [
+    {
+      "answer_count": 2,
+      "interface_pass": true,
+      "reorder_same_label": true,
+      "rename_same_description": true,
+      "selected_expected_description": true,
+      "outputs": [
+        {
+          "predicted_label": "fresh_choice_2_0",
+          "allowed_choices": [
+            "fresh_choice_2_0",
+            "fresh_choice_2_1"
+          ],
+          "candidates": [
+            {
+              "label": "fresh_choice_2_0",
+              "score": 0.8051604628562927,
+              "raw_score": -24.558311462402344,
+              "rank": 1
+            },
+            {
+              "label": "fresh_choice_2_1",
+              "score": 0.19483955204486847,
+              "raw_score": -25.977176666259766,
+              "rank": 2
+            }
+          ],
+          "truncated": false,
+          "max_sequence_length": 4096,
+          "note": "Scores rank this supplied choice set; they are not calibrated confidence."
+        },
+        {
+          "predicted_label": "fresh_choice_2_0",
+          "allowed_choices": [
+            "fresh_choice_2_1",
+            "fresh_choice_2_0"
+          ],
+          "candidates": [
+            {
+              "label": "fresh_choice_2_0",
+              "score": 0.8051604628562927,
+              "raw_score": -24.558311462402344,
+              "rank": 1
+            },
+            {
+              "label": "fresh_choice_2_1",
+              "score": 0.19483955204486847,
+              "raw_score": -25.977176666259766,
+              "rank": 2
+            }
+          ],
+          "truncated": false,
+          "max_sequence_length": 4096,
+          "note": "Scores rank this supplied choice set; they are not calibrated confidence."
+        },
+        {
+          "predicted_label": "opaque_2_0",
+          "allowed_choices": [
+            "opaque_2_0",
+            "opaque_2_1"
+          ],
+          "candidates": [
+            {
+              "label": "opaque_2_0",
+              "score": 0.7122752070426941,
+              "raw_score": -22.207761764526367,
+              "rank": 1
+            },
+            {
+              "label": "opaque_2_1",
+              "score": 0.2877248227596283,
+              "raw_score": -23.114221572875977,
+              "rank": 2
+            }
+          ],
+          "truncated": false,
+          "max_sequence_length": 4096,
+          "note": "Scores rank this supplied choice set; they are not calibrated confidence."
+        },
+        {
+          "predicted_label": "Escalate this damaged parcel to a human support agent.",
+          "allowed_choices": [
+            "Escalate this damaged parcel to a human support agent.",
+            "Search the product catalog for a new item."
+          ],
+          "candidates": [
+            {
+              "label": "Escalate this damaged parcel to a human support agent.",
+              "score": 0.7943422794342041,
+              "raw_score": -19.58828353881836,
+              "rank": 1
+            },
+            {
+              "label": "Search the product catalog for a new item.",
+              "score": 0.2056577205657959,
+              "raw_score": -20.939584732055664,
+              "rank": 2
+            }
+          ],
+          "truncated": false,
+          "max_sequence_length": 4096,
+          "note": "Scores rank this supplied choice set; they are not calibrated confidence."
+        }
+      ]
+    },
+    {
+      "answer_count": 3,
+      "interface_pass": true,
+      "reorder_same_label": true,
+      "rename_same_description": false,
+      "selected_expected_description": true,
+      "outputs": [
+        {
+          "predicted_label": "fresh_choice_3_0",
+          "allowed_choices": [
+            "fresh_choice_3_0",
+            "fresh_choice_3_1",
+            "fresh_choice_3_2"
+          ],
+          "candidates": [
+            {
+              "label": "fresh_choice_3_0",
+              "score": 0.5637261867523193,
+              "raw_score": -24.52130699157715,
+              "rank": 1
+            },
+            {
+              "label": "fresh_choice_3_2",
+              "score": 0.28383156657218933,
+              "raw_score": -25.207494735717773,
+              "rank": 2
+            },
+            {
+              "label": "fresh_choice_3_1",
+              "score": 0.15244220197200775,
+              "raw_score": -25.829090118408203,
+              "rank": 3
+            }
+          ],
+          "truncated": false,
+          "max_sequence_length": 4096,
+          "note": "Scores rank this supplied choice set; they are not calibrated confidence."
+        },
+        {
+          "predicted_label": "fresh_choice_3_0",
+          "allowed_choices": [
+            "fresh_choice_3_2",
+            "fresh_choice_3_1",
+            "fresh_choice_3_0"
+          ],
+          "candidates": [
+            {
+              "label": "fresh_choice_3_0",
+              "score": 0.563726544380188,
+              "raw_score": -24.52130699157715,
+              "rank": 1
+            },
+            {
+              "label": "fresh_choice_3_2",
+              "score": 0.2838311791419983,
+              "raw_score": -25.207496643066406,
+              "rank": 2
+            },
+            {
+              "label": "fresh_choice_3_1",
+              "score": 0.1524423062801361,
+              "raw_score": -25.829090118408203,
+              "rank": 3
+            }
+          ],
+          "truncated": false,
+          "max_sequence_length": 4096,
+          "note": "Scores rank this supplied choice set; they are not calibrated confidence."
+        },
+        {
+          "predicted_label": "opaque_3_2",
+          "allowed_choices": [
+            "opaque_3_0",
+            "opaque_3_1",
+            "opaque_3_2"
+          ],
+          "candidates": [
+            {
+              "label": "opaque_3_2",
+              "score": 0.5318363308906555,
+              "raw_score": -21.825939178466797,
+              "rank": 1
+            },
+            {
+              "label": "opaque_3_0",
+              "score": 0.34079551696777344,
+              "raw_score": -22.270992279052734,
+              "rank": 2
+            },
+            {
+              "label": "opaque_3_1",
+              "score": 0.12736809253692627,
+              "raw_score": -23.25519371032715,
+              "rank": 3
+            }
+          ],
+          "truncated": false,
+          "max_sequence_length": 4096,
+          "note": "Scores rank this supplied choice set; they are not calibrated confidence."
+        },
+        {
+          "predicted_label": "Route to unrelated department number 0 for a different request.",
+          "allowed_choices": [
+            "Escalate this damaged parcel to a human support agent.",
+            "Search the product catalog for a new item.",
+            "Route to unrelated department number 0 for a different request."
+          ],
+          "candidates": [
+            {
+              "label": "Route to unrelated department number 0 for a different request.",
+              "score": 0.6520880460739136,
+              "raw_score": -18.729812622070312,
+              "rank": 1
+            },
+            {
+              "label": "Escalate this damaged parcel to a human support agent.",
+              "score": 0.2763611972332001,
+              "raw_score": -19.58828353881836,
+              "rank": 2
+            },
+            {
+              "label": "Search the product catalog for a new item.",
+              "score": 0.07155078649520874,
+              "raw_score": -20.939584732055664,
+              "rank": 3
+            }
+          ],
+          "truncated": false,
+          "max_sequence_length": 4096,
+          "note": "Scores rank this supplied choice set; they are not calibrated confidence."
+        }
+      ]
+    },
+    {
+      "answer_count": 7,
+      "interface_pass": true,
+      "reorder_same_label": true,
+      "rename_same_description": false,
+      "selected_expected_description": true,
+      "outputs": [
+        {
+          "predicted_label": "fresh_choice_7_0",
+          "allowed_choices": [
+            "fresh_choice_7_0",
+            "fresh_choice_7_1",
+            "fresh_choice_7_2",
+            "fresh_choice_7_3",
+            "fresh_choice_7_4",
+            "fresh_choice_7_5",
+            "fresh_choice_7_6"
+          ],
+          "candidates": [
+            {
+              "label": "fresh_choice_7_0",
+              "score": 0.33595216274261475,
+              "raw_score": -23.554637908935547,
+              "rank": 1
+            },
+            {
+              "label": "fresh_choice_7_6",
+              "score": 0.2605591416358948,
+              "raw_score": -23.80877685546875,
+              "rank": 2
+            },
+            {
+              "label": "fresh_choice_7_4",
+              "score": 0.10382113605737686,
+              "raw_score": -24.72893714904785,
+              "rank": 3
+            },
+            {
+              "label": "fresh_choice_7_3",
+              "score": 0.08751508593559265,
+              "raw_score": -24.899795532226562,
+              "rank": 4
+            },
+            {
+              "label": "fresh_choice_7_5",
+              "score": 0.07923448085784912,
+              "raw_score": -24.999195098876953,
+              "rank": 5
+            },
+            {
+              "label": "fresh_choice_7_2",
+              "score": 0.07844322174787521,
+              "raw_score": -25.009231567382812,
+              "rank": 6
+            },
+            {
+              "label": "fresh_choice_7_1",
+              "score": 0.05447477847337723,
+              "raw_score": -25.373868942260742,
+              "rank": 7
+            }
+          ],
+          "truncated": false,
+          "max_sequence_length": 4096,
+          "note": "Scores rank this supplied choice set; they are not calibrated confidence."
+        },
+        {
+          "predicted_label": "fresh_choice_7_0",
+          "allowed_choices": [
+            "fresh_choice_7_6",
+            "fresh_choice_7_5",
+            "fresh_choice_7_4",
+            "fresh_choice_7_3",
+            "fresh_choice_7_2",
+            "fresh_choice_7_1",
+            "fresh_choice_7_0"
+          ],
+          "candidates": [
+            {
+              "label": "fresh_choice_7_0",
+              "score": 0.3359522819519043,
+              "raw_score": -23.554637908935547,
+              "rank": 1
+            },
+            {
+              "label": "fresh_choice_7_6",
+              "score": 0.26055923104286194,
+              "raw_score": -23.80877685546875,
+              "rank": 2
+            },
+            {
+              "label": "fresh_choice_7_4",
+              "score": 0.10382117331027985,
+              "raw_score": -24.72893714904785,
+              "rank": 3
+            },
+            {
+              "label": "fresh_choice_7_3",
+              "score": 0.08751478046178818,
+              "raw_score": -24.899799346923828,
+              "rank": 4
+            },
+            {
+              "label": "fresh_choice_7_5",
+              "score": 0.07923451066017151,
+              "raw_score": -24.999195098876953,
+              "rank": 5
+            },
+            {
+              "label": "fresh_choice_7_2",
+              "score": 0.0784432515501976,
+              "raw_score": -25.009231567382812,
+              "rank": 6
+            },
+            {
+              "label": "fresh_choice_7_1",
+              "score": 0.05447479709982872,
+              "raw_score": -25.373868942260742,
+              "rank": 7
+            }
+          ],
+          "truncated": false,
+          "max_sequence_length": 4096,
+          "note": "Scores rank this supplied choice set; they are not calibrated confidence."
+        },
+        {
+          "predicted_label": "opaque_7_6",
+          "allowed_choices": [
+            "opaque_7_0",
+            "opaque_7_1",
+            "opaque_7_2",
+            "opaque_7_3",
+            "opaque_7_4",
+            "opaque_7_5",
+            "opaque_7_6"
+          ],
+          "candidates": [
+            {
+              "label": "opaque_7_6",
+              "score": 0.3010425269603729,
+              "raw_score": -21.268993377685547,
+              "rank": 1
+            },
+            {
+              "label": "opaque_7_0",
+              "score": 0.22324608266353607,
+              "raw_score": -21.567970275878906,
+              "rank": 2
+            },
+            {
+              "label": "opaque_7_4",
+              "score": 0.114542156457901,
+              "raw_score": -22.235301971435547,
+              "rank": 3
+            },
+            {
+              "label": "opaque_7_2",
+              "score": 0.11444498598575592,
+              "raw_score": -22.23615074157715,
+              "rank": 4
+            },
+            {
+              "label": "opaque_7_3",
+              "score": 0.11412934213876724,
+              "raw_score": -22.23891258239746,
+              "rank": 5
+            },
+            {
+              "label": "opaque_7_5",
+              "score": 0.07986637204885483,
+              "raw_score": -22.595890045166016,
+              "rank": 6
+            },
+            {
+              "label": "opaque_7_1",
+              "score": 0.05272847041487694,
+              "raw_score": -23.011089324951172,
+              "rank": 7
+            }
+          ],
+          "truncated": false,
+          "max_sequence_length": 4096,
+          "note": "Scores rank this supplied choice set; they are not calibrated confidence."
+        },
+        {
+          "predicted_label": "Route to unrelated department number 0 for a different request.",
+          "allowed_choices": [
+            "Escalate this damaged parcel to a human support agent.",
+            "Search the product catalog for a new item.",
+            "Route to unrelated department number 0 for a different request.",
+            "Route to unrelated department number 1 for a different request.",
+            "Route to unrelated department number 2 for a different request.",
+            "Route to unrelated department number 3 for a different request.",
+            "Route to unrelated department number 4 for a different request."
+          ],
+          "candidates": [
+            {
+              "label": "Route to unrelated department number 0 for a different request.",
+              "score": 0.2675093412399292,
+              "raw_score": -18.729812622070312,
+              "rank": 1
+            },
+            {
+              "label": "Route to unrelated department number 4 for a different request.",
+              "score": 0.2286183089017868,
+              "raw_score": -18.886913299560547,
+              "rank": 2
+            },
+            {
+              "label": "Route to unrelated department number 1 for a different request.",
+              "score": 0.1698872447013855,
+              "raw_score": -19.1838321685791,
+              "rank": 3
+            },
+            {
+              "label": "Escalate this damaged parcel to a human support agent.",
+              "score": 0.1133730337023735,
+              "raw_score": -19.58828353881836,
+              "rank": 4
+            },
+            {
+              "label": "Route to unrelated department number 3 for a different request.",
+              "score": 0.10056313127279282,
+              "raw_score": -19.708181381225586,
+              "rank": 5
+            },
+            {
+              "label": "Route to unrelated department number 2 for a different request.",
+              "score": 0.09069626778364182,
+              "raw_score": -19.811450958251953,
+              "rank": 6
+            },
+            {
+              "label": "Search the product catalog for a new item.",
+              "score": 0.029352637007832527,
+              "raw_score": -20.939584732055664,
+              "rank": 7
+            }
+          ],
+          "truncated": false,
+          "max_sequence_length": 4096,
+          "note": "Scores rank this supplied choice set; they are not calibrated confidence."
+        }
+      ]
+    },
+    {
+      "answer_count": 20,
+      "interface_pass": true,
+      "reorder_same_label": true,
+      "rename_same_description": true,
+      "selected_expected_description": false,
+      "outputs": [
+        {
+          "predicted_label": "fresh_choice_20_11",
+          "allowed_choices": [
+            "fresh_choice_20_0",
+            "fresh_choice_20_1",
+            "fresh_choice_20_2",
+            "fresh_choice_20_3",
+            "fresh_choice_20_4",
+            "fresh_choice_20_5",
+            "fresh_choice_20_6",
+            "fresh_choice_20_7",
+            "fresh_choice_20_8",
+            "fresh_choice_20_9",
+            "fresh_choice_20_10",
+            "fresh_choice_20_11",
+            "fresh_choice_20_12",
+            "fresh_choice_20_13",
+            "fresh_choice_20_14",
+            "fresh_choice_20_15",
+            "fresh_choice_20_16",
+            "fresh_choice_20_17",
+            "fresh_choice_20_18",
+            "fresh_choice_20_19"
+          ],
+          "candidates": [
+            {
+              "label": "fresh_choice_20_11",
+              "score": 0.15572811663150787,
+              "raw_score": -24.59111785888672,
+              "rank": 1
+            },
+            {
+              "label": "fresh_choice_20_10",
+              "score": 0.1482214629650116,
+              "raw_score": -24.640522003173828,
+              "rank": 2
+            },
+            {
+              "label": "fresh_choice_20_0",
+              "score": 0.10967609286308289,
+              "raw_score": -24.94169807434082,
+              "rank": 3
+            },
+            {
+              "label": "fresh_choice_20_9",
+              "score": 0.09642379730939865,
+              "raw_score": -25.070476531982422,
+              "rank": 4
+            },
+            {
+              "label": "fresh_choice_20_6",
+              "score": 0.062188707292079926,
+              "raw_score": -25.509056091308594,
+              "rank": 5
+            },
+            {
+              "label": "fresh_choice_20_8",
+              "score": 0.057998985052108765,
+              "raw_score": -25.57880401611328,
+              "rank": 6
+            },
+            {
+              "label": "fresh_choice_20_7",
+              "score": 0.05253210663795471,
+              "raw_score": -25.677804946899414,
+              "rank": 7
+            },
+            {
+              "label": "fresh_choice_20_13",
+              "score": 0.038757286965847015,
+              "raw_score": -25.981910705566406,
+              "rank": 8
+            },
+            {
+              "label": "fresh_choice_20_14",
+              "score": 0.03664031997323036,
+              "raw_score": -26.0380802154541,
+              "rank": 9
+            },
+            {
+              "label": "fresh_choice_20_12",
+              "score": 0.03644376993179321,
+              "raw_score": -26.043458938598633,
+              "rank": 10
+            },
+            {
+              "label": "fresh_choice_20_2",
+              "score": 0.026935890316963196,
+              "raw_score": -26.34576988220215,
+              "rank": 11
+            },
+            {
+              "label": "fresh_choice_20_4",
+              "score": 0.024683654308319092,
+              "raw_score": -26.433088302612305,
+              "rank": 12
+            },
+            {
+              "label": "fresh_choice_20_5",
+              "score": 0.024211952462792397,
+              "raw_score": -26.452383041381836,
+              "rank": 13
+            },
+            {
+              "label": "fresh_choice_20_19",
+              "score": 0.023727986961603165,
+              "raw_score": -26.47257423400879,
+              "rank": 14
+            },
+            {
+              "label": "fresh_choice_20_3",
+              "score": 0.022604424506425858,
+              "raw_score": -26.52108383178711,
+              "rank": 15
+            },
+            {
+              "label": "fresh_choice_20_1",
+              "score": 0.021840117871761322,
+              "raw_score": -26.55548095703125,
+              "rank": 16
+            },
+            {
+              "label": "fresh_choice_20_15",
+              "score": 0.02026970498263836,
+              "raw_score": -26.630102157592773,
+              "rank": 17
+            },
+            {
+              "label": "fresh_choice_20_16",
+              "score": 0.019944682717323303,
+              "raw_score": -26.64626693725586,
+              "rank": 18
+            },
+            {
+              "label": "fresh_choice_20_18",
+              "score": 0.011263599619269371,
+              "raw_score": -27.217653274536133,
+              "rank": 19
+            },
+            {
+              "label": "fresh_choice_20_17",
+              "score": 0.00990742165595293,
+              "raw_score": -27.345945358276367,
+              "rank": 20
+            }
+          ],
+          "truncated": false,
+          "max_sequence_length": 4096,
+          "note": "Scores rank this supplied choice set; they are not calibrated confidence."
+        },
+        {
+          "predicted_label": "fresh_choice_20_11",
+          "allowed_choices": [
+            "fresh_choice_20_19",
+            "fresh_choice_20_18",
+            "fresh_choice_20_17",
+            "fresh_choice_20_16",
+            "fresh_choice_20_15",
+            "fresh_choice_20_14",
+            "fresh_choice_20_13",
+            "fresh_choice_20_12",
+            "fresh_choice_20_11",
+            "fresh_choice_20_10",
+            "fresh_choice_20_9",
+            "fresh_choice_20_8",
+            "fresh_choice_20_7",
+            "fresh_choice_20_6",
+            "fresh_choice_20_5",
+            "fresh_choice_20_4",
+            "fresh_choice_20_3",
+            "fresh_choice_20_2",
+            "fresh_choice_20_1",
+            "fresh_choice_20_0"
+          ],
+          "candidates": [
+            {
+              "label": "fresh_choice_20_11",
+              "score": 0.15572810173034668,
+              "raw_score": -24.59111785888672,
+              "rank": 1
+            },
+            {
+              "label": "fresh_choice_20_10",
+              "score": 0.1482214480638504,
+              "raw_score": -24.640522003173828,
+              "rank": 2
+            },
+            {
+              "label": "fresh_choice_20_0",
+              "score": 0.10967607796192169,
+              "raw_score": -24.94169807434082,
+              "rank": 3
+            },
+            {
+              "label": "fresh_choice_20_9",
+              "score": 0.09642378240823746,
+              "raw_score": -25.070476531982422,
+              "rank": 4
+            },
+            {
+              "label": "fresh_choice_20_6",
+              "score": 0.06218870356678963,
+              "raw_score": -25.509056091308594,
+              "rank": 5
+            },
+            {
+              "label": "fresh_choice_20_8",
+              "score": 0.057998981326818466,
+              "raw_score": -25.57880401611328,
+              "rank": 6
+            },
+            {
+              "label": "fresh_choice_20_7",
+              "score": 0.05253210291266441,
+              "raw_score": -25.677804946899414,
+              "rank": 7
+            },
+            {
+              "label": "fresh_choice_20_13",
+              "score": 0.03875728324055672,
+              "raw_score": -25.981910705566406,
+              "rank": 8
+            },
+            {
+              "label": "fresh_choice_20_14",
+              "score": 0.036640316247940063,
+              "raw_score": -26.0380802154541,
+              "rank": 9
+            },
+            {
+              "label": "fresh_choice_20_12",
+              "score": 0.036443766206502914,
+              "raw_score": -26.043458938598633,
+              "rank": 10
+            },
+            {
+              "label": "fresh_choice_20_2",
+              "score": 0.026935888454318047,
+              "raw_score": -26.34576988220215,
+              "rank": 11
+            },
+            {
+              "label": "fresh_choice_20_4",
+              "score": 0.024683650583028793,
+              "raw_score": -26.433088302612305,
+              "rank": 12
+            },
+            {
+              "label": "fresh_choice_20_5",
+              "score": 0.024211950600147247,
+              "raw_score": -26.452383041381836,
+              "rank": 13
+            },
+            {
+              "label": "fresh_choice_20_19",
+              "score": 0.023727985098958015,
+              "raw_score": -26.47257423400879,
+              "rank": 14
+            },
+            {
+              "label": "fresh_choice_20_3",
+              "score": 0.02260442264378071,
+              "raw_score": -26.52108383178711,
+              "rank": 15
+            },
+            {
+              "label": "fresh_choice_20_1",
+              "score": 0.021840116009116173,
+              "raw_score": -26.55548095703125,
+              "rank": 16
+            },
+            {
+              "label": "fresh_choice_20_15",
+              "score": 0.02026970311999321,
+              "raw_score": -26.630102157592773,
+              "rank": 17
+            },
+            {
+              "label": "fresh_choice_20_16",
+              "score": 0.019944680854678154,
+              "raw_score": -26.64626693725586,
+              "rank": 18
+            },
+            {
+              "label": "fresh_choice_20_18",
+              "score": 0.011263598687946796,
+              "raw_score": -27.217653274536133,
+              "rank": 19
+            },
+            {
+              "label": "fresh_choice_20_17",
+              "score": 0.009907420724630356,
+              "raw_score": -27.345945358276367,
+              "rank": 20
+            }
+          ],
+          "truncated": false,
+          "max_sequence_length": 4096,
+          "note": "Scores rank this supplied choice set; they are not calibrated confidence."
+        },
+        {
+          "predicted_label": "opaque_20_11",
+          "allowed_choices": [
+            "opaque_20_0",
+            "opaque_20_1",
+            "opaque_20_2",
+            "opaque_20_3",
+            "opaque_20_4",
+            "opaque_20_5",
+            "opaque_20_6",
+            "opaque_20_7",
+            "opaque_20_8",
+            "opaque_20_9",
+            "opaque_20_10",
+            "opaque_20_11",
+            "opaque_20_12",
+            "opaque_20_13",
+            "opaque_20_14",
+            "opaque_20_15",
+            "opaque_20_16",
+            "opaque_20_17",
+            "opaque_20_18",
+            "opaque_20_19"
+          ],
+          "candidates": [
+            {
+              "label": "opaque_20_11",
+              "score": 0.17951472103595734,
+              "raw_score": -21.678247451782227,
+              "rank": 1
+            },
+            {
+              "label": "opaque_20_10",
+              "score": 0.16386361420154572,
+              "raw_score": -21.76947021484375,
+              "rank": 2
+            },
+            {
+              "label": "opaque_20_9",
+              "score": 0.1559082716703415,
+              "raw_score": -21.819236755371094,
+              "rank": 3
+            },
+            {
+              "label": "opaque_20_8",
+              "score": 0.06722499430179596,
+              "raw_score": -22.660459518432617,
+              "rank": 4
+            },
+            {
+              "label": "opaque_20_6",
+              "score": 0.061009395867586136,
+              "raw_score": -22.757476806640625,
+              "rank": 5
+            },
+            {
+              "label": "opaque_20_0",
+              "score": 0.05371956154704094,
+              "raw_score": -22.884727478027344,
+              "rank": 6
+            },
+            {
+              "label": "opaque_20_7",
+              "score": 0.046380579471588135,
+              "raw_score": -23.03162384033203,
+              "rank": 7
+            },
+            {
+              "label": "opaque_20_2",
+              "score": 0.031455062329769135,
+              "raw_score": -23.419944763183594,
+              "rank": 8
+            },
+            {
+              "label": "opaque_20_12",
+              "score": 0.0313202403485775,
+              "raw_score": -23.424240112304688,
+              "rank": 9
+            },
+            {
+              "label": "opaque_20_14",
+              "score": 0.029301172122359276,
+              "raw_score": -23.490877151489258,
+              "rank": 10
+            },
+            {
+              "label": "opaque_20_13",
+              "score": 0.026141302660107613,
+              "raw_score": -23.60498809814453,
+              "rank": 11
+            },
+            {
+              "label": "opaque_20_19",
+              "score": 0.02265744097530842,
+              "raw_score": -23.748016357421875,
+              "rank": 12
+            },
+            {
+              "label": "opaque_20_4",
+              "score": 0.022092893719673157,
+              "raw_score": -23.77324867248535,
+              "rank": 13
+            },
+            {
+              "label": "opaque_20_3",
+              "score": 0.02106008678674698,
+              "raw_score": -23.821125030517578,
+              "rank": 14
+            },
+            {
+              "label": "opaque_20_5",
+              "score": 0.019397132098674774,
+              "raw_score": -23.903379440307617,
+              "rank": 15
+            },
+            {
+              "label": "opaque_20_1",
+              "score": 0.018865037709474564,
+              "raw_score": -23.931194305419922,
+              "rank": 16
+            },
+            {
+              "label": "opaque_20_15",
+              "score": 0.016952816396951675,
+              "raw_score": -24.038070678710938,
+              "rank": 17
+            },
+            {
+              "label": "opaque_20_16",
+              "score": 0.013765859417617321,
+              "raw_score": -24.246313095092773,
+              "rank": 18
+            },
+            {
+              "label": "opaque_20_18",
+              "score": 0.012271336279809475,
+              "raw_score": -24.361238479614258,
+              "rank": 19
+            },
+            {
+              "label": "opaque_20_17",
+              "score": 0.007098542992025614,
+              "raw_score": -24.908615112304688,
+              "rank": 20
+            }
+          ],
+          "truncated": false,
+          "max_sequence_length": 4096,
+          "note": "Scores rank this supplied choice set; they are not calibrated confidence."
+        },
+        {
+          "predicted_label": "Route to unrelated department number 9 for a different request.",
+          "allowed_choices": [
+            "Escalate this damaged parcel to a human support agent.",
+            "Search the product catalog for a new item.",
+            "Route to unrelated department number 0 for a different request.",
+            "Route to unrelated department number 1 for a different request.",
+            "Route to unrelated department number 2 for a different request.",
+            "Route to unrelated department number 3 for a different request.",
+            "Route to unrelated department number 4 for a different request.",
+            "Route to unrelated department number 5 for a different request.",
+            "Route to unrelated department number 6 for a different request.",
+            "Route to unrelated department number 7 for a different request.",
+            "Route to unrelated department number 8 for a different request.",
+            "Route to unrelated department number 9 for a different request.",
+            "Route to unrelated department number 10 for a different request.",
+            "Route to unrelated department number 11 for a different request.",
+            "Route to unrelated department number 12 for a different request.",
+            "Route to unrelated department number 13 for a different request.",
+            "Route to unrelated department number 14 for a different request.",
+            "Route to unrelated department number 15 for a different request.",
+            "Route to unrelated department number 16 for a different request.",
+            "Route to unrelated department number 17 for a different request."
+          ],
+          "candidates": [
+            {
+              "label": "Route to unrelated department number 9 for a different request.",
+              "score": 0.11579183489084244,
+              "raw_score": -18.278705596923828,
+              "rank": 1
+            },
+            {
+              "label": "Route to unrelated department number 8 for a different request.",
+              "score": 0.11046068370342255,
+              "raw_score": -18.32583999633789,
+              "rank": 2
+            },
+            {
+              "label": "Route to unrelated department number 11 for a different request.",
+              "score": 0.09375981986522675,
+              "raw_score": -18.489763259887695,
+              "rank": 3
+            },
+            {
+              "label": "Route to unrelated department number 7 for a different request.",
+              "score": 0.07637739926576614,
+              "raw_score": -18.694812774658203,
+              "rank": 4
+            },
+            {
+              "label": "Route to unrelated department number 0 for a different request.",
+              "score": 0.07375044375658035,
+              "raw_score": -18.729812622070312,
+              "rank": 5
+            },
+            {
+              "label": "Route to unrelated department number 10 for a different request.",
+              "score": 0.06551573425531387,
+              "raw_score": -18.848209381103516,
+              "rank": 6
+            },
+            {
+              "label": "Route to unrelated department number 4 for a different request.",
+              "score": 0.06302846223115921,
+              "raw_score": -18.886913299560547,
+              "rank": 7
+            },
+            {
+              "label": "Route to unrelated department number 6 for a different request.",
+              "score": 0.051630470901727676,
+              "raw_score": -19.086387634277344,
+              "rank": 8
+            },
+            {
+              "label": "Route to unrelated department number 12 for a different request.",
+              "score": 0.04935718700289726,
+              "raw_score": -19.13141632080078,
+              "rank": 9
+            },
+            {
+              "label": "Route to unrelated department number 1 for a different request.",
+              "score": 0.046836718916893005,
+              "raw_score": -19.1838321685791,
+              "rank": 10
+            },
+            {
+              "label": "Route to unrelated department number 16 for a different request.",
+              "score": 0.03468938171863556,
+              "raw_score": -19.484066009521484,
+              "rank": 11
+            },
+            {
+              "label": "Route to unrelated department number 13 for a different request.",
+              "score": 0.03264395520091057,
+              "raw_score": -19.54483985900879,
+              "rank": 12
+            },
+            {
+              "label": "Escalate this damaged parcel to a human support agent.",
+              "score": 0.03125615045428276,
+              "raw_score": -19.58828353881836,
+              "rank": 13
+            },
+            {
+              "label": "Route to unrelated department number 14 for a different request.",
+              "score": 0.03116500936448574,
+              "raw_score": -19.591203689575195,
+              "rank": 14
+            },
+            {
+              "label": "Route to unrelated department number 5 for a different request.",
+              "score": 0.02856743521988392,
+              "raw_score": -19.678232192993164,
+              "rank": 15
+            },
+            {
+              "label": "Route to unrelated department number 3 for a different request.",
+              "score": 0.027724549174308777,
+              "raw_score": -19.708181381225586,
+              "rank": 16
+            },
+            {
+              "label": "Route to unrelated department number 2 for a different request.",
+              "score": 0.025004321709275246,
+              "raw_score": -19.811450958251953,
+              "rank": 17
+            },
+            {
+              "label": "Route to unrelated department number 15 for a different request.",
+              "score": 0.02304059825837612,
+              "raw_score": -19.89324188232422,
+              "rank": 18
+            },
+            {
+              "label": "Route to unrelated department number 17 for a different request.",
+              "score": 0.011307581327855587,
+              "raw_score": -20.605026245117188,
+              "rank": 19
+            },
+            {
+              "label": "Search the product catalog for a new item.",
+              "score": 0.008092314936220646,
+              "raw_score": -20.939584732055664,
+              "rank": 20
+            }
+          ],
+          "truncated": false,
+          "max_sequence_length": 4096,
+          "note": "Scores rank this supplied choice set; they are not calibrated confidence."
+        }
+      ]
+    }
+  ]
+}

evaluation/per-task-baselines.json

@@ -0,0 +1,67 @@
+{
+  "dataset_revision": "f2fb14e4ec977c420f376c08785664cd38763d7e",
+  "families": {
+    "agent_next_action_type": {
+      "n": 1158,
+      "majority_correct": 616,
+      "majority_accuracy": 0.531951640759931,
+      "majority_labels": [
+        "text_response"
+      ],
+      "candidate_count": {
+        "median": 3.0,
+        "min": 3,
+        "max": 3
+      },
+      "uniform_expected_accuracy": 0.3333333333333333,
+      "reference_definition": "Descriptive test-set constant-label majority; no model fitting",
+      "unseen_task_family": false,
+      "training_fixed_majority": {
+        "label": "text_response",
+        "correct": 616,
+        "n": 1158,
+        "majority_label_absent_from_choices": 0
+      }
+    },
+    "tool_selection": {
+      "n": 542,
+      "majority_correct": 79,
+      "majority_accuracy": 0.14575645756457564,
+      "majority_labels": [
+        "transfer_to_human_agent"
+      ],
+      "candidate_count": {
+        "median": 20.0,
+        "min": 11,
+        "max": 32
+      },
+      "uniform_expected_accuracy": 0.051859254651728436,
+      "reference_definition": "Descriptive test-set constant-label majority; no model fitting",
+      "unseen_task_family": false,
+      "training_fixed_majority": {
+        "label": "transfer_to_human_agent",
+        "correct": 79,
+        "n": 542,
+        "majority_label_absent_from_choices": 235
+      }
+    },
+    "when_to_call_tool": {
+      "n": 3652,
+      "majority_correct": 1295,
+      "majority_accuracy": 0.3546002190580504,
+      "majority_labels": [
+        "cannot_answer",
+        "tool_call"
+      ],
+      "candidate_count": {
+        "median": 4.0,
+        "min": 4,
+        "max": 4
+      },
+      "uniform_expected_accuracy": 0.25,
+      "reference_definition": "Descriptive test-set constant-label majority; no model fitting",
+      "unseen_task_family": true
+    }
+  },
+  "when2call_training_rows": 0
+}

evaluation/when2call-results.json

@@ -0,0 +1,11 @@
+{
+  "task": "when_to_call_tool",
+  "correct": 1256,
+  "n": 3652,
+  "accuracy": 0.343921139101862,
+  "majority_reference": 0.3546002190580504,
+  "lift_percentage_points": -1.0679079956188386,
+  "unseen_task_family": true,
+  "checkpoint_identity": "cd9b0bfaea164d5ad186bca3a4b4aaf3cb4896c18fae14c488be597afa82f0cc",
+  "truncated_rows": 0
+}

example-output.json

@@ -0,0 +1,24 @@
+{
+  "predicted_label": "lookup_order",
+  "allowed_choices": [
+    "lookup_order",
+    "search_catalog"
+  ],
+  "candidates": [
+    {
+      "label": "lookup_order",
+      "score": 0.7431679368019104,
+      "raw_score": -13.1875,
+      "rank": 1
+    },
+    {
+      "label": "search_catalog",
+      "score": 0.25683197379112244,
+      "raw_score": -14.25,
+      "rank": 2
+    }
+  ],
+  "truncated": false,
+  "max_sequence_length": 4096,
+  "note": "Scores rank this supplied choice set; they are not calibrated confidence."
+}

metrics.jsonl

@@ -0,0 +1,74 @@
+{"event": "env", "mode": "prototype", "device": "cuda", "gpu": "NVIDIA A100-SXM4-80GB", "torch": "2.12.0+cu126", "transformers": "5.17.0", "attention": "kernels-community/flash-attn2@f50dc99ed079b35990bc895d43fd353ea0cb376d", "ts": 1790774736.9}
+{"event": "training_api_validated", "mode": "prototype", "ts": 1790774737.1}
+{"event": "data_prep", "n_rows": 60000, "n_a": 40809, "n_t": 19191, "family_counts": {"agent_next_action_type": 40809, "tool_selection": 19191}, "gold_class_counts": {"agent_next_action_type": {"tool_call": 19173, "text_response": 21636}, "tool_selection": {"distinct": 2840}}, "selected_row_ids_sha256": "c5b306b0471ba104161051a7241b3bce4b69e1d959ff3e34fb06ef8eb4b077d9", "seconds": 25.0, "ts": 1790774762.8}
+{"event": "lazy_pairs_ready", "tag": "train", "n_rows": 60000, "n_pairs": 199191, "max_len": 4096, "mapped_pairs": 0, "ts": 1790774763.4}
+{"event": "prototype_start", "pool": "sampled4", "n_rows_selected": 60000, "n_a": 40809, "n_t": 19191, "ts": 1790774763.9}
+{"event": "lazy_pairs_ready", "tag": "eval", "n_rows": 599, "n_pairs": 4767, "max_len": 4096, "mapped_pairs": 0, "ts": 1790774765.7}
+{"event": "coverage_progress", "step": 1, "rows_seen": 17, "target": 60000, "elapsed_seconds": 44.8, "ts": 1790774781.7}
+{"event": "coverage_progress", "step": 2, "rows_seen": 33, "target": 60000, "elapsed_seconds": 46.2, "ts": 1790774783.0}
+{"event": "coverage_progress", "step": 3, "rows_seen": 49, "target": 60000, "elapsed_seconds": 47.5, "ts": 1790774784.4}
+{"event": "coverage_progress", "step": 4, "rows_seen": 65, "target": 60000, "elapsed_seconds": 48.9, "ts": 1790774785.7}
+{"event": "coverage_progress", "step": 5, "rows_seen": 81, "target": 60000, "elapsed_seconds": 50.4, "ts": 1790774787.2}
+{"event": "coverage_progress", "step": 100, "rows_seen": 1618, "target": 60000, "elapsed_seconds": 226.6, "ts": 1790774963.4}
+{"event": "coverage_progress", "step": 200, "rows_seen": 3228, "target": 60000, "elapsed_seconds": 414.1, "ts": 1790775150.9}
+{"event": "coverage_progress", "step": 300, "rows_seen": 4853, "target": 60000, "elapsed_seconds": 602.0, "ts": 1790775338.8}
+{"event": "coverage_progress", "step": 400, "rows_seen": 6455, "target": 60000, "elapsed_seconds": 787.4, "ts": 1790775524.2}
+{"event": "val_acc", "step": 400, "n_rows": 599, "acc": 0.5008347245409015, "n": 599, "macro_accuracy": 0.44902728815772297, "per_family": {"agent_next_action_type": {"acc": 0.5845410628019324, "n": 414}, "tool_selection": {"acc": 0.31351351351351353, "n": 185}}, "n_pairs": 4767, "n_truncated_pairs": 158, "seconds": 86.2, "ts": 1790775610.4}
+{"event": "coverage_progress", "step": 500, "rows_seen": 8078, "target": 60000, "elapsed_seconds": 1059.2, "ts": 1790775796.0}
+{"event": "coverage_progress", "step": 600, "rows_seen": 9706, "target": 60000, "elapsed_seconds": 1250.8, "ts": 1790775987.7}
+{"event": "coverage_progress", "step": 700, "rows_seen": 11313, "target": 60000, "elapsed_seconds": 1436.6, "ts": 1790776173.5}
+{"event": "coverage_progress", "step": 800, "rows_seen": 12925, "target": 60000, "elapsed_seconds": 1623.0, "ts": 1790776359.8}
+{"event": "val_acc", "step": 800, "n_rows": 599, "acc": 0.5709515859766278, "n": 599, "macro_accuracy": 0.5147016581799191, "per_family": {"agent_next_action_type": {"acc": 0.6618357487922706, "n": 414}, "tool_selection": {"acc": 0.3675675675675676, "n": 185}}, "n_pairs": 4767, "n_truncated_pairs": 158, "seconds": 86.8, "ts": 1790776446.6}
+{"event": "coverage_progress", "step": 900, "rows_seen": 14539, "target": 60000, "elapsed_seconds": 1896.8, "ts": 1790776633.6}
+{"event": "coverage_progress", "step": 1000, "rows_seen": 16165, "target": 60000, "elapsed_seconds": 2084.6, "ts": 1790776821.4}
+{"event": "coverage_progress", "step": 1100, "rows_seen": 17789, "target": 60000, "elapsed_seconds": 2274.0, "ts": 1790777010.8}
+{"event": "coverage_progress", "step": 1200, "rows_seen": 19410, "target": 60000, "elapsed_seconds": 2463.7, "ts": 1790777200.5}
+{"event": "val_acc", "step": 1200, "n_rows": 599, "acc": 0.6010016694490818, "n": 599, "macro_accuracy": 0.557370413892153, "per_family": {"agent_next_action_type": {"acc": 0.6714975845410628, "n": 414}, "tool_selection": {"acc": 0.44324324324324327, "n": 185}}, "n_pairs": 4767, "n_truncated_pairs": 158, "seconds": 86.5, "ts": 1790777287.0}
+{"event": "coverage_progress", "step": 1300, "rows_seen": 21023, "target": 60000, "elapsed_seconds": 2741.0, "ts": 1790777477.9}
+{"event": "coverage_progress", "step": 1400, "rows_seen": 22641, "target": 60000, "elapsed_seconds": 2927.7, "ts": 1790777664.6}
+{"event": "coverage_progress", "step": 1500, "rows_seen": 24251, "target": 60000, "elapsed_seconds": 3116.2, "ts": 1790777853.0}
+{"event": "coverage_progress", "step": 1600, "rows_seen": 25854, "target": 60000, "elapsed_seconds": 3308.0, "ts": 1790778044.9}
+{"event": "val_acc", "step": 1600, "n_rows": 599, "acc": 0.6060100166944908, "n": 599, "macro_accuracy": 0.5654785220002612, "per_family": {"agent_next_action_type": {"acc": 0.6714975845410628, "n": 414}, "tool_selection": {"acc": 0.4594594594594595, "n": 185}}, "n_pairs": 4767, "n_truncated_pairs": 158, "seconds": 87.7, "ts": 1790778132.6}
+{"event": "coverage_progress", "step": 1700, "rows_seen": 27460, "target": 60000, "elapsed_seconds": 3581.6, "ts": 1790778318.4}
+{"event": "coverage_progress", "step": 1800, "rows_seen": 29067, "target": 60000, "elapsed_seconds": 3769.7, "ts": 1790778506.6}
+{"event": "coverage_progress", "step": 1900, "rows_seen": 30690, "target": 60000, "elapsed_seconds": 3959.4, "ts": 1790778696.2}
+{"event": "coverage_progress", "step": 2000, "rows_seen": 32306, "target": 60000, "elapsed_seconds": 4151.0, "ts": 1790778887.8}
+{"event": "val_acc", "step": 2000, "n_rows": 599, "acc": 0.6227045075125208, "n": 599, "macro_accuracy": 0.5880206293249772, "per_family": {"agent_next_action_type": {"acc": 0.678743961352657, "n": 414}, "tool_selection": {"acc": 0.4972972972972973, "n": 185}}, "n_pairs": 4767, "n_truncated_pairs": 158, "seconds": 87.0, "ts": 1790778975.7}
+{"event": "coverage_progress", "step": 2100, "rows_seen": 33913, "target": 60000, "elapsed_seconds": 4425.4, "ts": 1790779162.2}
+{"event": "coverage_progress", "step": 2200, "rows_seen": 35510, "target": 60000, "elapsed_seconds": 4610.9, "ts": 1790779347.7}
+{"event": "coverage_progress", "step": 2300, "rows_seen": 37119, "target": 60000, "elapsed_seconds": 4800.3, "ts": 1790779537.2}
+{"event": "coverage_progress", "step": 2400, "rows_seen": 38723, "target": 60000, "elapsed_seconds": 4986.8, "ts": 1790779723.7}
+{"event": "val_acc", "step": 2400, "n_rows": 599, "acc": 0.659432387312187, "n": 599, "macro_accuracy": 0.6340253296775036, "per_family": {"agent_next_action_type": {"acc": 0.7004830917874396, "n": 414}, "tool_selection": {"acc": 0.5675675675675675, "n": 185}}, "n_pairs": 4767, "n_truncated_pairs": 158, "seconds": 86.8, "ts": 1790779810.5}
+{"event": "coverage_progress", "step": 2500, "rows_seen": 40337, "target": 60000, "elapsed_seconds": 5263.2, "ts": 1790780000.0}
+{"event": "coverage_progress", "step": 2600, "rows_seen": 41950, "target": 60000, "elapsed_seconds": 5454.1, "ts": 1790780191.0}
+{"event": "coverage_progress", "step": 2700, "rows_seen": 43568, "target": 60000, "elapsed_seconds": 5643.4, "ts": 1790780380.3}
+{"event": "coverage_progress", "step": 2800, "rows_seen": 45191, "target": 60000, "elapsed_seconds": 5831.1, "ts": 1790780567.9}
+{"event": "val_acc", "step": 2800, "n_rows": 599, "acc": 0.657762938230384, "n": 599, "macro_accuracy": 0.6253427340383862, "per_family": {"agent_next_action_type": {"acc": 0.7101449275362319, "n": 414}, "tool_selection": {"acc": 0.5405405405405406, "n": 185}}, "n_pairs": 4767, "n_truncated_pairs": 158, "seconds": 88.0, "ts": 1790780655.9}
+{"event": "coverage_progress", "step": 2900, "rows_seen": 46798, "target": 60000, "elapsed_seconds": 6106.0, "ts": 1790780842.9}
+{"event": "coverage_progress", "step": 3000, "rows_seen": 48405, "target": 60000, "elapsed_seconds": 6295.0, "ts": 1790781031.8}
+{"event": "coverage_progress", "step": 3100, "rows_seen": 50021, "target": 60000, "elapsed_seconds": 6482.9, "ts": 1790781219.8}
+{"event": "coverage_progress", "step": 3200, "rows_seen": 51627, "target": 60000, "elapsed_seconds": 6671.1, "ts": 1790781407.9}
+{"event": "val_acc", "step": 3200, "n_rows": 599, "acc": 0.6944908180300501, "n": 599, "macro_accuracy": 0.6638725682203943, "per_family": {"agent_next_action_type": {"acc": 0.7439613526570048, "n": 414}, "tool_selection": {"acc": 0.5837837837837838, "n": 185}}, "n_pairs": 4767, "n_truncated_pairs": 158, "seconds": 86.7, "ts": 1790781494.7}
+{"event": "coverage_progress", "step": 3300, "rows_seen": 53230, "target": 60000, "elapsed_seconds": 6949.0, "ts": 1790781685.9}
+{"event": "coverage_progress", "step": 3400, "rows_seen": 54841, "target": 60000, "elapsed_seconds": 7135.8, "ts": 1790781872.6}
+{"event": "coverage_progress", "step": 3500, "rows_seen": 56445, "target": 60000, "elapsed_seconds": 7323.8, "ts": 1790782060.7}
+{"event": "coverage_progress", "step": 3600, "rows_seen": 58065, "target": 60000, "elapsed_seconds": 7513.4, "ts": 1790782250.3}
+{"event": "val_acc", "step": 3600, "n_rows": 599, "acc": 0.671118530884808, "n": 599, "macro_accuracy": 0.6394894894894895, "per_family": {"agent_next_action_type": {"acc": 0.7222222222222222, "n": 414}, "tool_selection": {"acc": 0.5567567567567567, "n": 185}}, "n_pairs": 4767, "n_truncated_pairs": 158, "seconds": 87.9, "ts": 1790782338.2}
+{"event": "coverage_progress", "step": 3700, "rows_seen": 59683, "target": 60000, "elapsed_seconds": 7790.1, "ts": 1790782527.0}
+{"event": "train_done", "pool": "sampled4", "seconds": 7786.5, "steps": 3720, "rows_covered": 60000, "n_rows_selected": 60000, "n_pairs_processed": 199191, "n_truncated_pairs": 9249, "note": "rows_covered counts unique decisions actually iterated; no full-epoch guarantee", "ts": 1790782565.0}
+{"event": "checkpoint_saved_before_evaluation", "rows_covered": 60000, "ts": 1790782579.5}
+{"event": "lazy_pairs_ready", "tag": "eval", "n_rows": 1798, "n_pairs": 14449, "max_len": 4096, "mapped_pairs": 0, "ts": 1790782579.5}
+{"event": "val_full", "acc": 0.6840934371523916, "n": 1798, "macro_accuracy": 0.6529034180751321, "per_family": {"agent_next_action_type": {"acc": 0.7348912167606769, "n": 1241}, "tool_selection": {"acc": 0.5709156193895871, "n": 557}}, "n_pairs": 14449, "n_truncated_pairs": 443, "seconds": 266.6, "ts": 1790782846.2}
+{"event": "test_prep", "n_rows": 1700, "skipped": 0, "ts": 1790782848.2}
+{"event": "lazy_pairs_ready", "tag": "eval", "n_rows": 1700, "n_pairs": 14251, "max_len": 4096, "mapped_pairs": 0, "ts": 1790782848.2}
+{"event": "final_test", "acc": 0.6982352941176471, "n": 1700, "macro_accuracy": 0.6778976986661058, "per_family": {"agent_next_action_type": {"acc": 0.7340241796200345, "n": 1158}, "tool_selection": {"acc": 0.6217712177121771, "n": 542}}, "n_pairs": 14251, "n_truncated_pairs": 414, "seconds": 274.1, "ts": 1790783122.4}
+{"event": "lazy_pairs_ready", "tag": "eval", "n_rows": 299, "n_pairs": 2523, "max_len": 4096, "mapped_pairs": 0, "ts": 1790783122.4}
+{"event": "lazy_pairs_ready", "tag": "eval", "n_rows": 299, "n_pairs": 2523, "max_len": 4096, "mapped_pairs": 0, "ts": 1790783172.5}
+{"event": "lazy_pairs_ready", "tag": "eval", "n_rows": 299, "n_pairs": 2523, "max_len": 4096, "mapped_pairs": 0, "ts": 1790783222.7}
+{"event": "lazy_pairs_ready", "tag": "eval", "n_rows": 299, "n_pairs": 2523, "max_len": 4096, "mapped_pairs": 0, "ts": 1790783272.9}
+{"event": "lazy_pairs_ready", "tag": "eval", "n_rows": 299, "n_pairs": 2523, "max_len": 4096, "mapped_pairs": 0, "ts": 1790783323.2}
+{"event": "lazy_pairs_ready", "tag": "eval", "n_rows": 299, "n_pairs": 2523, "max_len": 4096, "mapped_pairs": 0, "ts": 1790783373.7}
+{"event": "shuffled_invariance", "invariance_rate": 1.0, "perms": 5, "n_rows": 299, "ts": 1790783423.8}
+{"event": "baselines", "val": {"uniform_expected": {"acc": 0.24665407679797663, "n": 1798, "per_family": {"agent_next_action_type": {"acc": 0.33333333333332843, "n": 1241}, "tool_selection": {"acc": 0.05353207076499354, "n": 557}}, "macro_accuracy": 0.193432702049161}, "train_frequency": {"acc": 0.47052280311457173, "n": 1798, "per_family": {"agent_next_action_type": {"acc": 0.5511684125705076, "n": 1241}, "tool_selection": {"acc": 0.29084380610412924, "n": 557}}, "macro_accuracy": 0.4210061093373184}}, "test": {"uniform_expected": {"acc": 0.24359277413013672, "n": 1700, "per_family": {"agent_next_action_type": {"acc": 0.33333333333332943, "n": 1158}, "tool_selection": {"acc": 0.051859254651728665, "n": 542}}, "macro_accuracy": 0.19259629399252903}, "train_frequency": {"acc": 0.43529411764705883, "n": 1700, "per_family": {"agent_next_action_type": {"acc": 0.531951640759931, "n": 1158}, "tool_selection": {"acc": 0.22878228782287824, "n": 542}}, "macro_accuracy": 0.3803669642914046}}, "ts": 1790783423.9}
+{"event": "baseline_untrained_head", "val": {"acc": 0.2925472747497219, "n": 1798, "macro_accuracy": 0.21439969214610907, "per_family": {"agent_next_action_type": {"acc": 0.41982272360999195, "n": 1241}, "tool_selection": {"acc": 0.008976660682226212, "n": 557}}, "n_pairs": 14449, "n_truncated_pairs": 443, "seconds": 267.2}, "test": {"acc": 0.29411764705882354, "n": 1700, "macro_accuracy": 0.2178523857777438, "per_family": {"agent_next_action_type": {"acc": 0.4283246977547496, "n": 1158}, "tool_selection": {"acc": 0.007380073800738007, "n": 542}}, "n_pairs": 14251, "n_truncated_pairs": 414, "seconds": 274.4}, "ts": 1790783965.7}
+{"event": "latency_gpu", "measurement": "one candidate forward pass, excludes tokenization", "warmup_pairs": 5, "pair_ms_p50": 23.6, "pair_ms_p95": 24.2, "ts": 1790783969.6}

model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd9b0bfaea164d5ad186bca3a4b4aaf3cb4896c18fae14c488be597afa82f0cc
+size 598436708

per-task-results.json

@@ -0,0 +1,95 @@
+{
+  "checkpoint_revision": "5623e421346eeaad6a1c7abe959edc4e1f5875e3",
+  "weights_sha256": "cd9b0bfaea164d5ad186bca3a4b4aaf3cb4896c18fae14c488be597afa82f0cc",
+  "dataset_revision": "f2fb14e4ec977c420f376c08785664cd38763d7e",
+  "training_rows": 60000,
+  "optimizer_steps": 3720,
+  "tasks": {
+    "agent_next_action_type": {
+      "correct": 850,
+      "n": 1158,
+      "accuracy": 0.7340241796200345,
+      "constant_majority_reference": 0.531951640759931,
+      "majority_correct": 616,
+      "lift_percentage_points": 20.207253886010356,
+      "candidate_count": {
+        "median": 3.0,
+        "min": 3,
+        "max": 3
+      },
+      "unseen_task_family": false,
+      "untrained_modernbert_scalar_head_accuracy": 0.4283246977547496,
+      "allowed_choice_training_frequency_accuracy": 0.531951640759931,
+      "uniform_expected_accuracy": 0.3333333333333333
+    },
+    "tool_selection": {
+      "correct": 337,
+      "n": 542,
+      "accuracy": 0.6217712177121771,
+      "constant_majority_reference": 0.14575645756457564,
+      "majority_correct": 79,
+      "lift_percentage_points": 47.601476014760145,
+      "candidate_count": {
+        "median": 20.0,
+        "min": 11,
+        "max": 32
+      },
+      "unseen_task_family": false,
+      "untrained_modernbert_scalar_head_accuracy": 0.007380073800738007,
+      "allowed_choice_training_frequency_accuracy": 0.22878228782287824,
+      "uniform_expected_accuracy": 0.051859254651728436
+    },
+    "when_to_call_tool": {
+      "correct": 1256,
+      "n": 3652,
+      "accuracy": 0.343921139101862,
+      "constant_majority_reference": 0.3546002190580504,
+      "majority_correct": 1295,
+      "lift_percentage_points": -1.0679079956188386,
+      "candidate_count": {
+        "median": 4.0,
+        "min": 4,
+        "max": 4
+      },
+      "unseen_task_family": true,
+      "uniform_expected_accuracy": 0.25
+    }
+  },
+  "open_label_checks": [
+    {
+      "answer_count": 2,
+      "interface_pass": true,
+      "reorder_same_label": true,
+      "rename_same_description": true,
+      "selected_expected_description": true
+    },
+    {
+      "answer_count": 3,
+      "interface_pass": true,
+      "reorder_same_label": true,
+      "rename_same_description": false,
+      "selected_expected_description": true
+    },
+    {
+      "answer_count": 7,
+      "interface_pass": true,
+      "reorder_same_label": true,
+      "rename_same_description": false,
+      "selected_expected_description": true
+    },
+    {
+      "answer_count": 20,
+      "interface_pass": true,
+      "reorder_same_label": true,
+      "rename_same_description": true,
+      "selected_expected_description": false
+    }
+  ],
+  "candidate_order_check": {
+    "invariance_rate": 1.0,
+    "perms": 5,
+    "n_rows": 299
+  },
+  "cost_estimate_including_failed_jobs_usd": 8.7919,
+  "invoice_verified": false
+}

pilot/checkpoint-reload-smoke.json

@@ -0,0 +1,34 @@
+{
+  "pilot_only": true,
+  "coverage_not_prototype": true,
+  "checkpoint_parameter_equality": true,
+  "same_precision_logit_equality": true,
+  "float32_score": 0.1755414605140686,
+  "bf16_score": 0.1669921875,
+  "absolute_precision_difference": 0.008549273014068604,
+  "inference_helper_returned_valid_choice": true,
+  "illustrative_prediction": {
+    "predicted_label": "search_catalog",
+    "allowed_choices": [
+      "lookup_order",
+      "search_catalog"
+    ],
+    "candidates": [
+      {
+        "label": "search_catalog",
+        "score": 0.5236632227897644,
+        "raw_score": -0.6127176284790039,
+        "rank": 1
+      },
+      {
+        "label": "lookup_order",
+        "score": 0.476336807012558,
+        "raw_score": -0.7074412107467651,
+        "rank": 2
+      }
+    ],
+    "truncated": false,
+    "max_sequence_length": 4096,
+    "note": "Scores rank this supplied choice set; they are not calibrated confidence."
+  }
+}

pilot/feasibility.json

@@ -0,0 +1,7 @@
+{
+  "remaining_seconds": 4407,
+  "training_allowance_seconds": 2607,
+  "measured_rows_per_s": 18.72,
+  "estimated_training_seconds": 3205.1282051282055,
+  "full_60000_feasible": false
+}

pilot/pilot_metrics.jsonl

@@ -0,0 +1,5 @@
+{"event": "env", "mode": "pilot", "device": "cuda", "gpu": "NVIDIA H200", "torch": "2.12.0+cu126", "transformers": "5.17.0", "attention": "kernels-community/flash-attn2@f50dc99ed079b35990bc895d43fd353ea0cb376d", "ts": 1790772005.8}
+{"event": "data_prep", "n_rows": 500, "n_a": 340, "n_t": 160, "family_counts": {"agent_next_action_type": 340, "tool_selection": 160}, "gold_class_counts": {"agent_next_action_type": {"text_response": 192, "tool_call": 148}, "tool_selection": {"distinct": 97}}, "selected_row_ids_sha256": "fc9bcbbbfdbfee3d6df4bf7d56a9b658829e70ff7db3039d64fb1bf8add77587", "seconds": 12.8, "ts": 1790772020.4}
+{"event": "tokenize", "tag": "train", "n_pairs": 4205, "n_at_max": 236, "n_truncated": 236, "len_p50": 2548, "len_p95": 4096, "len_max": 4096, "seconds": 6.9, "ts": 1790772027.7}
+{"event": "pilot_phase", "pool": "sampled4", "steps": 40, "seconds": 28.85, "steps_per_s": 1.386, "rows_seen": 500, "rows_per_s": 18.72, "pairs_seen": 1789, "pairs_per_s": 62.0, "tokens_per_s": 165721, "max_mem_gb": 80.82, "ts": 1790772065.7}
+{"event": "latency_gpu", "measurement": "one candidate forward pass, excludes tokenization", "warmup_pairs": 5, "pair_ms_p50": 14.8, "pair_ms_p95": 22.0, "ts": 1790772068.2}

pilot/pilot_results.json

@@ -0,0 +1,27 @@
+{
+  "phases": {
+    "sampled4": {
+      "steps": 40,
+      "seconds": 28.85,
+      "steps_per_s": 1.386,
+      "rows_seen": 500,
+      "rows_per_s": 18.72,
+      "pairs_seen": 1789,
+      "pairs_per_s": 62.0,
+      "tokens_per_s": 165721,
+      "max_mem_gb": 80.82
+    }
+  },
+  "latency": {
+    "measurement": "one candidate forward pass, excludes tokenization",
+    "warmup_pairs": 5,
+    "pair_ms_p50": 14.8,
+    "pair_ms_p95": 22.0
+  },
+  "env": {
+    "gpu": "NVIDIA H200",
+    "torch": "2.12.0+cu126"
+  },
+  "checkpoint_parameter_equality": true,
+  "checkpoint_reload_verified": true
+}

predict.py

@@ -0,0 +1,91 @@
+"""Typed-choice inference for ModernJEV-Decide-Preview.
+The encoder scores each declared (state, candidate) pair; it does not generate text.
+"""
+import json
+import os
+import torch
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+MAX_LEN = 4096
+MODEL_ID = "OpenMed/ModernJEV-Decide-Preview"
+
+def serialize_state(state):
+    if not isinstance(state, dict):
+        raise TypeError("state must be a dict containing conversation and available_tools")
+    conv = state.get("conversation") or []
+    policy = state.get("policy")
+    first = conv[0] if conv else None
+    duplicate = (policy is not None and isinstance(first, dict)
+                 and first.get("role") == "system" and first.get("content") == policy)
+    compact = {"available_tools": state.get("available_tools") or [], "conversation": conv}
+    if policy is not None and not duplicate:
+        compact["policy"] = policy
+    return json.dumps(compact, ensure_ascii=False)
+
+def normalize_criteria(criteria):
+    if isinstance(criteria, list):
+        if not criteria or any(not isinstance(v, str) or not v.strip() for v in criteria):
+            raise ValueError("Answer list must contain nonempty strings")
+        if len(set(criteria)) != len(criteria):
+            raise ValueError("Answer list must contain unique strings")
+        criteria = {v: v for v in criteria}
+    if not isinstance(criteria, dict) or not criteria:
+        raise ValueError("criteria must be a nonempty mapping or list of unique answer strings")
+    if any(not isinstance(k, str) or not k.strip() or not isinstance(v, str)
+           for k, v in criteria.items()):
+        raise TypeError("Choice labels must be nonempty strings and descriptions must be strings")
+    return criteria
+
+class DecisionModel:
+    def __init__(self, model_path=MODEL_ID, device=None, revision=None):
+        self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
+        self.tokenizer = AutoTokenizer.from_pretrained(model_path, revision=revision)
+        self.model = AutoModelForSequenceClassification.from_pretrained(
+            model_path, revision=revision, attn_implementation="sdpa").to(self.device).eval()
+        if self.model.config.num_labels != 1:
+            raise ValueError("Expected a trained scalar candidate-scoring head")
+
+    @torch.inference_mode()
+    def decide(self, *, state, question, criteria, candidate_batch_size=8):
+        if not isinstance(question, str) or not question.strip():
+            raise ValueError("question must be nonempty text")
+        criteria = normalize_criteria(criteria)
+        if not isinstance(candidate_batch_size, int) or isinstance(candidate_batch_size, bool) or candidate_batch_size < 1:
+            raise ValueError("candidate_batch_size must be a positive integer")
+        keys = list(criteria)
+        text_a = question + "\n\nSTATE:\n" + serialize_state(state)
+        text_bs = [f"{k}: {criteria[k]}" for k in keys]
+        raw_a_length = len(self.tokenizer(text_a, add_special_tokens=False, verbose=False)["input_ids"])
+        special = self.tokenizer.num_special_tokens_to_add(pair=True)
+        raw_lengths = [raw_a_length + len(self.tokenizer(t, add_special_tokens=False)["input_ids"]) + special for t in text_bs]
+        scores = []
+        for begin in range(0, len(keys), candidate_batch_size):
+            ts = text_bs[begin:begin + candidate_batch_size]
+            encoded = self.tokenizer([text_a] * len(ts), ts, truncation="only_first",
+                max_length=MAX_LEN, padding=True, return_tensors="pt", verbose=False).to(self.device)
+            with torch.autocast(device_type=self.device.split(":")[0],
+                                dtype=torch.bfloat16, enabled=self.device.startswith("cuda")):
+                logits = self.model(input_ids=encoded["input_ids"],
+                                    attention_mask=encoded["attention_mask"]).logits.squeeze(-1)
+            scores.extend(logits.float().cpu().tolist())
+        probabilities = torch.softmax(torch.tensor(scores), dim=0).tolist()
+        order = sorted(range(len(keys)), key=lambda i: (-scores[i], keys[i]))
+        return {"predicted_label": keys[order[0]], "allowed_choices": keys,
+            "candidates": [{"label": keys[i], "score": probabilities[i], "raw_score": scores[i],
+                            "rank": rank + 1} for rank,i in enumerate(order)],
+            "truncated": any(n > MAX_LEN for n in raw_lengths),
+            "max_sequence_length": MAX_LEN,
+            "note": "Scores rank this supplied choice set; they are not calibrated confidence."}
+
+_default = None
+def predict_typed(question_text, state_json, criteria_json):
+    global _default
+    if _default is None:
+        _default = DecisionModel(os.environ.get("MODEL_PATH", MODEL_ID))
+    state = json.loads(state_json) if isinstance(state_json, str) else state_json
+    criteria = json.loads(criteria_json) if isinstance(criteria_json, str) else criteria_json
+    return _default.decide(state=state, question=question_text, criteria=criteria)
+
+def predict_batch(rows):
+    return [predict_typed(r["question_text"], r["state_json"], r["criteria_json"]) for r in rows]
+

recipe/finalize.py

@@ -0,0 +1,41 @@
+import argparse,json,os,time,importlib.util,importlib.metadata
+from pathlib import Path
+from huggingface_hub import HfApi
+p=argparse.ArgumentParser();p.add_argument('--run_dir',required=True);p.add_argument('--repo',default='OpenMed/ModernJEV-Decide-Preview');args=p.parse_args()
+folder=Path(args.run_dir);results=json.loads((folder/'results.json').read_text())
+assert results['test']['n']==1700
+spec=importlib.util.spec_from_file_location('predict_helper',folder/'predict.py');helper=importlib.util.module_from_spec(spec);spec.loader.exec_module(helper)
+client=helper.DecisionModel(str(folder/'model'))
+state={'policy':'Use the order lookup tool when a customer asks about an order.','conversation':[{'role':'user','content':'Where is order A123?'}],'available_tools':[{'name':'lookup_order','description':"Retrieve an order's delivery status."},{'name':'search_catalog','description':'Find products in the catalog.'}]}
+criteria={'lookup_order':"Retrieve the delivery status of the customer's order.",'search_catalog':'Search for products that match a shopping request.'}
+example=client.decide(state=state,question='Which tool should the assistant call next?',criteria=criteria)
+assert example['predicted_label'] in criteria and all(c['label'] in criteria for c in example['candidates'])
+(folder/'example-output.json').write_text(json.dumps(example,indent=2)+'\n')
+versions={name:importlib.metadata.version(name) for name in ['torch','transformers','datasets','accelerate','huggingface-hub','kernels']}
+(folder/'runtime-versions.json').write_text(json.dumps(versions,indent=2)+'\n')
+card="---\nlicense: apache-2.0\nlibrary_name: transformers\npipeline_tag: text-classification\nbase_model: answerdotai/ModernBERT-base\nbase_model_relation: finetune\ndatasets:\n  - MaziyarPanahi/AgentToolDecisions-180K\ntags:\n  - modernbert\n  - encoder\n  - decision-model\n  - tool-routing\n  - agentic\n  - preview\nlanguage:\n  - en\n---\n\n<div align=\"center\">\n\n# ModernJEV-Decide-Preview\n\n### Give your agent a next move.\n\n**A small encoder for choosing an action or a tool from the options you provide.**\n\n149.6M parameters · 4,096-token inputs · Typed choices · Built on ModernBERT\n\n</div>\n\n> **Training preview.** Training and evaluation are pending. This draft contains no measured accuracy or speed claims. The final card will report actual training coverage and held-out results.\n\nAn agent doesn't need to write a paragraph every time it makes a decision. Sometimes the useful output is simply **answer**, **call a tool**, or **choose this tool**.\n\nModernJEV-Decide-Preview reads a conversation, its policy and available tools, then scores the choices supplied with your question. Your application receives one of those labels, along with the scores for the alternatives. The model ranks choices; your application executes the selected action.\n\nThe architecture is a **ModernBERT cross-encoder with one scalar scoring head**. Choice labels and descriptions are input text, so the output is not limited to a fixed vocabulary of tool names. This is a Jev-style choice model built from public agent decisions, with its own training and evaluation; it is not a reproduction of Jev.\n\n## Where it fits\n\n| Use case | Supply | Receive |\n|---|---|---|\n| **Next-action routing** | Current conversation, policy, tool list | A declared action such as `text_response` or `tool_call` |\n| **Tool selection** | The task and each available tool's purpose | The label of the selected tool |\n| **Workflow branching** | A text state and clearly described alternatives | One branch label; evaluate on your own workflow before adoption |\n| **Decision-model experiments** | Your choices and held-out tasks | Candidate rankings you can inspect and compare |\n\nUseful places to start: support agents choosing between lookup and escalation, assistants selecting an API, and workflows choosing between a direct answer and a tool-backed step. The model does not generate tool arguments or execute tools.\n\n## Quick start\n\nInstall PyTorch and Transformers, then use the included inference helper so the input formatting matches training:\n\n```bash\npip install torch transformers huggingface-hub\n```\n\nDownload the supplied helper, then run it alongside your application. During the private preview, your Hugging Face account must have access to the repository.\n\n```bash\nhf download OpenMed/ModernJEV-Decide-Preview predict.py --local-dir modernjev\ncd modernjev\n```\n\nThe helper's input contract has been reviewed. The examples' predictions will be recorded after the trained checkpoint passes evaluation.\n\n```python\nfrom predict import DecisionModel\n\nmodel = DecisionModel(\"OpenMed/ModernJEV-Decide-Preview\")\ndecision = model.decide(\n    state={\n        \"policy\": \"Use the order lookup tool when a customer asks about an order.\",\n        \"conversation\": [\n            {\"role\": \"user\", \"content\": \"Where is order A123?\"}\n        ],\n        \"available_tools\": [\n            {\"name\": \"lookup_order\", \"description\": \"Retrieve an order's delivery status.\"},\n            {\"name\": \"search_catalog\", \"description\": \"Find products in the catalog.\"},\n        ],\n    },\n    question=\"Which tool should the assistant call next?\",\n    criteria={\n        \"lookup_order\": \"Retrieve the delivery status of the customer's order.\",\n        \"search_catalog\": \"Search for products that match a shopping request.\",\n    },\n)\nprint(decision[\"predicted_label\"])\nprint(decision[\"candidates\"])\n```\n\nThe result contains the selected label, all candidate scores, and whether input truncation occurred. Scores are normalized **within the supplied choice set**; they are not calibrated confidence estimates and cannot be compared directly across unrelated requests.\n\n## Training recipe\n\n| Setting | Planned configuration |\n|---|---|\n| Base | `answerdotai/ModernBERT-base` |\n| Base revision | `8949b909ec900327062f0ebf497f51aef5e6f0c8` |\n| Parameters with scalar head | 149,605,633 |\n| Input limit | 4,096 tokens including both sequences and special tokens |\n| Dataset | `MaziyarPanahi/AgentToolDecisions-180K` |\n| Dataset revision | `f2fb14e4ec977c420f376c08785664cd38763d7e` |\n| Selected training target | Exactly 60,000 train-split decisions, stratified by family, seed 42 |\n| Task families | `agent_next_action_type`, `tool_selection` |\n| Objective | Per-decision softmax cross-entropy over candidate scores |\n| Candidate grouping | `row_id`, preserving `group_id` only for episode/split boundaries |\n| Candidate pool | Pilot will choose full or sampled declared alternatives; final recipe records which ran |\n| Evaluation | Monitor official validation; evaluate the final fixed-epoch checkpoint on 1,700 in-scope official test decisions |\n\nThe dataset has 180,000 total rows. This prototype targets 60,000 of its 112,973 in-scope training decisions; **180,000 is not the number used for training**. Exact selected row IDs, coverage, steps, dependency versions and measured cost will accompany the checkpoint.\n\n## Evaluation\n\n**Pending.** The final comparison will include:\n\n| System | Next action | Tool selection | Macro accuracy |\n|---|---|---|---|\n| ModernJEV-Decide-Preview | Pending | Pending | Pending |\n| ModernBERT + untrained scalar head, seed 42 | Pending | Pending | Pending |\n| Training-derived allowed-choice frequency baseline | Pending | Pending | Pending |\n| Uniform over supplied choices, expected accuracy | Pending | Pending | Pending |\n\nUntrained ModernBERT is a pretrained encoder, not a pretrained decision classifier. Its randomly initialized scalar head is a starting-point comparison, not a claim about ModernBERT's quality on other tasks. A frozen-backbone trained-head comparison will be reported separately if completed within the run budget.\n\nThe report will also include candidate-order checks, input truncation, per-family sample counts, measured inference latency, and actual training coverage. The published split audit found no cross-split conflicts for episode IDs or full upstream source identities.\n\n## Input design matters\n\n- Give every choice a concrete, distinct description. Include the same available tool information the application actually has.\n- Use the supplied serialization helper. Long state may need truncation at the 4,096-token limit; the helper reports it.\n- The training data is English agent conversations and tool decisions. New domains and arbitrary workflow labels need their own evaluation.\n- `tool_and_response` is a declared next-action choice but never a gold label in this dataset. That action is outside demonstrated positive training coverage.\n- This preview implements **choice ranking**. It does not implement Jev's other primitives, vision inputs, generated explanations, or autonomous tool execution.\n\n## Make your own\n\nThe package includes a reusable **HuggingChat ML Intern prompt**, the pinned training recipe and prediction examples. Change the namespace to your own account or organization, review the compute proposal, grant your budget, and keep your own validation/test data separate from training.\n\n## Attribution\n\nBase model: [Answer.AI ModernBERT](https://huggingface.co/answerdotai/ModernBERT-base), Apache 2.0.\n\nDecision dataset: [MaziyarPanahi/AgentToolDecisions-180K](https://huggingface.co/datasets/MaziyarPanahi/AgentToolDecisions-180K), transformed from the upstream agentic sources documented in its card. Keep the pinned revisions and source attribution with derivative work, and follow the upstream dataset licenses when redistributing data.\n\n**Workflow:** HuggingChat ML Intern prepared the proposal, training code, helper and model-card draft. Codex audited the data and corrected candidate targets, padding budgets, coverage accounting and checkpoint verification. Hugging Face Jobs was launched with the operator's approved local HF credential after the connector's write permissions returned 403. This is an assisted training workflow; HuggingChat did not autonomously execute the training job.\n"
+status=f"> **Experimental preview.** The final checkpoint processed **{results['rows_covered']:,}/{results['n_rows_selected']:,} selected training decisions**. Held-out results below cover all **1,700** in-scope official test decisions. This is a small choice-ranking prototype, not a Jev reproduction."
+card=card.replace('> **Training preview.** Training and evaluation are pending. This draft contains no measured accuracy or speed claims. The final card will report actual training coverage and held-out results.',status)
+if results['rows_covered'] != results['n_rows_selected']: card=card.replace(status,status+'\n\n> The time limit stopped training before a complete epoch. Selected rows are not the same as trained rows.')
+card=card.replace('The helper\'s input contract has been reviewed. The examples\' predictions will be recorded after the trained checkpoint passes evaluation.','The included helper was executed against this checkpoint. See [the observed example output](example-output.json); the snippet below prints the actual prediction rather than promising a fixed answer.')
+card=card.replace('| Setting | Planned configuration |','| Setting | Recorded configuration |')
+card=card.replace('| Selected training target | Exactly 60,000 train-split decisions, stratified by family, seed 42 |',f"| Selected training target | Exactly 60,000 train-split decisions, stratified by family, seed 42 |\n| Actual training coverage | {results['rows_covered']:,} decisions; {results['steps']:,} optimizer steps |\n| Attention backend | {results['attention']} |\n| Training elapsed | {results['train_seconds']/60:.1f} minutes |\n| GPU | {results['gpu']} |")
+card=card.replace('| Candidate pool | Pilot will choose full or sampled declared alternatives; final recipe records which ran |','| Candidate pool | Gold plus up to three declared negatives per training decision; evaluation ranks every declared choice |')
+systems=[('ModernJEV-Decide-Preview',results['test']),('ModernBERT + untrained scalar head, seed 42',results['baseline_untrained_head']['test']),('Allowed-choice training-frequency baseline',results['baselines']['test']['train_frequency']),('Uniform over allowed choices, expected accuracy',results['baselines']['test']['uniform_expected'])]
+evaluation='## Evaluation\n\nOfficial held-out test decisions. Next action: **1,158** cases. Tool selection: **542** cases. Macro accuracy gives both families equal weight.\n\n| System | Next action | Tool selection | Macro accuracy | Overall accuracy |\n|---|---|---|---|---|\n'
+for name,r in systems:
+    evaluation+=f"| {name} | {r['per_family']['agent_next_action_type']['acc']*100:.2f}% | {r['per_family']['tool_selection']['acc']*100:.2f}% | {r['macro_accuracy']*100:.2f}% | {r['acc']*100:.2f}% |\n"
+inv=results['shuffled_invariance'];lat=results['latency_gpu']
+evaluation+=f"\nCandidate-order label agreement: **{inv['invariance_rate']*100:.2f}%** across {inv['perms']} permutations of {inv['n_rows']} held-out decisions. Ties use lexical label order.\n\nGPU latency for **one candidate forward pass**, excluding tokenization: median {lat['pair_ms_p50']:.1f} ms, p95 {lat['pair_ms_p95']:.1f} ms. A full decision may contain several candidates; these are not end-to-end decision timings.\n\n"
+evaluation+='Untrained ModernBERT is a pretrained encoder with a randomly initialized scoring head, not a pretrained decision classifier. No frozen-backbone trained-head comparison or Jev API comparison was completed in this bounded run.\n\n'
+evaluation+='See [results.json](results.json), [training coverage](training_coverage.json), [metrics](metrics.jsonl), [runtime versions](runtime-versions.json) and [the pinned recipe](recipe/train.py). Input truncation is measured in the tokenization events in the metrics file. The published split audit found no cross-split episode or full upstream source conflicts.\n\n'
+start=card.index('## Evaluation');end=card.index('## Input design matters',start);card=card[:start]+evaluation+card[end:]
+card=card.replace('Exact selected row IDs, coverage, steps, dependency versions and measured cost will accompany the checkpoint.','Exact selected row IDs, coverage, steps and dependency versions accompany this checkpoint. This repository contains model artifacts; it does not expose an inference endpoint.')
+card=card.replace('## Where it fits','![How decisions are scored](architecture.svg)\n\n## Where it fits')
+(folder/'README.md').write_text(card)
+svg='<svg xmlns="http://www.w3.org/2000/svg" width="1080" height="220" viewBox="0 0 1080 220"><rect width="1080" height="220" rx="22" fill="#101820"/><g font-family="sans-serif" text-anchor="middle"><text x="540" y="40" font-size="18" fill="#a8d9bd">READ THE STATE. SCORE THE CHOICES. RETURN A LABEL.</text><g fill="#20332d" stroke="#4e9f79"><rect x="30" y="80" width="245" height="100" rx="12"/><rect x="350" y="80" width="350" height="100" rx="12"/><rect x="775" y="80" width="275" height="100" rx="12"/></g><g font-size="20" fill="#f1f4f2"><text x="152" y="117">STATE + QUESTION</text><text x="152" y="150">DECLARED CHOICES</text><text x="525" y="117">SHARED MODERNBERT</text><text x="525" y="150">CANDIDATE SCORER</text><text x="912" y="117">SELECTED LABEL</text><text x="912" y="150">+ CHOICE RANKINGS</text></g><g fill="#8bd9af" font-size="32"><text x="311" y="140">→</text><text x="738" y="140">→</text></g></g></svg>'
+(folder/'architecture.svg').write_text(svg)
+api=HfApi();assert api.model_info(args.repo).private
+for filename in ['README.md','architecture.svg','runtime-versions.json','example-output.json']:
+    api.upload_file(path_or_fileobj=str(folder/filename),path_in_repo=filename,repo_id=args.repo,repo_type='model',commit_message='Measured prototype results and verified quick start')
+print('MODEL_CARD_FINALIZED '+json.dumps({'rows_covered':results['rows_covered'],'test_accuracy':results['test']['acc'],'private':True}),flush=True)

recipe/subset-60000.json

@@ -0,0 +1,17 @@
+{
+  "target_training_decisions": 60000,
+  "seed": 42,
+  "dataset_revision": "f2fb14e4ec977c420f376c08785664cd38763d7e",
+  "counts": {
+    "agent_next_action_type": 40809,
+    "tool_selection": 19191
+  },
+  "selected_row_ids_sha256": "c5b306b0471ba104161051a7241b3bce4b69e1d959ff3e34fb06ef8eb4b077d9",
+  "full_candidate_pairs": 504012,
+  "sampled4_candidate_pairs": 199191,
+  "next_action_gold_labels": {
+    "text_response": 21636,
+    "tool_call": 19173
+  },
+  "training_started": false
+}

recipe/train.py

@@ -0,0 +1,859 @@
+"""ModernJEV-Decide-Preview — training, evaluation, persistence.
+
+Dataset : MaziyarPanahi/AgentToolDecisions-180K @ f2fb14e4ec977c420f376c08785664cd38763d7e
+Base    : answerdotai/ModernBERT-base @ 8949b909ec900327062f0ebf497f51aef5e6f0c8
+Scope   : task_family in {agent_next_action_type, tool_selection} ONLY (choice primitive).
+
+Objective: shared-encoder candidate scalar scorer; per-row_id softmax cross-entropy
+over the row's DECLARED candidates. group_id is the EPISODE, not the decision —
+grouping is ALWAYS by row_id (asserted). Candidates enter the softmax as text
+(label + criterion), so variable tool names need no fixed head and no candidate
+index is exposed to the model. Inputs contain NO gold_label / gold_json /
+gold_score / label_source / source metadata.
+
+Training pool (--pool):
+  full     — every declared candidate of the row joins the softmax.
+  sampled4 — gold + up to 3 declared negatives, deterministic per (SEED, epoch,
+             row_id). This is ordinary sampled-choice softmax within the same loss;
+             EVALUATION ALWAYS RANKS ALL DECLARED CANDIDATES regardless of --pool.
+The pilot benchmarks both objectives and the launch report states which one ran.
+
+Modes:
+  pilot     — benchmarks full vs sampled4 throughput/memory, GPU latency, save check.
+  prototype — time-guarded training on the exact stratified subset, interval monitoring
+              on a stratified subset of the OFFICIAL validation split (full-val eval
+              recorded for the final fixed-epoch checkpoint), ONE final test evaluation on all in-scope
+              test rows, baselines (uniform / train-frequency / untrained ModernBERT
+              head), shuffled-order label invariance, optional frozen-backbone probe,
+              and persistence to --save_dir (default /output/modernjev). No Hub push
+              unless --push is passed explicitly.
+
+No Space is created anywhere; metrics are metrics.jsonl + stdout only.
+"""
+import argparse
+import gzip
+import hashlib
+import json
+import os
+import random
+import time
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from datasets import Dataset, load_dataset
+from torch.utils.data import DataLoader, Dataset as TorchDataset, Sampler
+from transformers import (AutoModelForSequenceClassification, AutoTokenizer,
+                          Trainer, TrainerCallback, TrainingArguments)
+
+DS_ID = "MaziyarPanahi/AgentToolDecisions-180K"
+DS_REV = "f2fb14e4ec977c420f376c08785664cd38763d7e"
+BASE_ID = "answerdotai/ModernBERT-base"
+BASE_REV = "8949b909ec900327062f0ebf497f51aef5e6f0c8"
+FOCUS = ("agent_next_action_type", "tool_selection")
+SEED = 42
+MAX_LEN = 4096
+ATTN_IMPL = os.environ.get("MODERNJEV_ATTN", "sdpa")
+EXPECTED = {"train": 171056, "validation": 2713, "test": 6231}
+EXPECTED_FOCUS_TRAIN = 112973
+
+METRICS = []
+
+
+def log_metric(d):
+    d["ts"] = round(time.time(), 1)
+    METRICS.append(d)
+    print("METRIC " + json.dumps(d, default=str), flush=True)
+
+
+def save_metrics(path):
+    with open(path, "w") as f:
+        for d in METRICS:
+            f.write(json.dumps(d, default=str) + "\n")
+
+
+def serialize_state(row):
+    """Compact the state. Drops the policy key ONLY when it exactly equals the
+    first system message (audited dedupe rule)."""
+    state = json.loads(row["state_json"])
+    conv = state.get("conversation") or []
+    policy = state.get("policy")
+    first = conv[0] if conv else None
+    dup = (policy is not None and isinstance(first, dict)
+           and first.get("role") == "system" and first.get("content") == policy)
+    compact = {"available_tools": state.get("available_tools") or [],
+               "conversation": conv}
+    if policy is not None and not dup:
+        compact["policy"] = policy
+    return json.dumps(compact, ensure_ascii=False)
+
+
+def parse_row(r):
+    criteria = json.loads(r["criteria_json"])
+    keys = list(criteria.keys())
+    gold = r["gold_label"]
+    return {"row_id": r["row_id"], "group_id": r["group_id"],
+            "family": r["task_family"],
+            "text_a": r["question_text"] + "\n\nSTATE:\n" + serialize_state(r),
+            "cand_keys": keys,
+            "cand_texts": [f"{k}: {criteria[k]}" for k in keys],
+            "gold_idx": keys.index(gold) if gold in keys else None}
+
+
+def load_and_prepare(n_rows, seed=SEED):
+    t0 = time.time()
+    ds = load_dataset(DS_ID, revision=DS_REV)
+    for split, n in EXPECTED.items():
+        assert len(ds[split]) == n, f"{split}: {len(ds[split])} != {n}"
+    train = ds["train"].filter(lambda r: r["task_family"] in FOCUS, num_proc=8)
+    assert len(train) == EXPECTED_FOCUS_TRAIN, f"{len(train)} != {EXPECTED_FOCUS_TRAIN}"
+
+    metadata = train.select_columns(["row_id", "task_family"])[:]
+    fam_idx = {f: [] for f in FOCUS}
+    for i, fam in enumerate(metadata["task_family"]):
+        fam_idx[fam].append(i)
+    n_a = n_rows * len(fam_idx["agent_next_action_type"]) // len(train)
+    n_t = n_rows - n_a
+    rng = random.Random(seed)
+    picked = []
+    for fam, k in (("agent_next_action_type", n_a), ("tool_selection", n_t)):
+        all_ids = metadata["row_id"]
+        idxs = sorted(fam_idx[fam], key=lambda i: all_ids[i])
+        picked += rng.sample(idxs, k)
+    assert len(picked) == n_rows
+    picked.sort()
+    fields = ["row_id", "group_id", "task_family", "question_text", "state_json", "criteria_json", "gold_label"]
+    selected = train.select(picked).select_columns(fields)[:]
+    parsed = [parse_row(dict(zip(fields, values))) for values in zip(*(selected[f] for f in fields))]
+    for r in parsed:
+        assert r["gold_idx"] is not None, f"gold missing in {r['row_id']}"
+
+    fam_counts = {f: sum(1 for r in parsed if r["family"] == f) for f in FOCUS}
+    gold_classes = {f: {} for f in FOCUS}
+    for r in parsed:
+        g = r["cand_keys"][r["gold_idx"]]
+        gold_classes[r["family"]][g] = gold_classes[r["family"]].get(g, 0) + 1
+    ids_hash = hashlib.sha256(
+        "\n".join(sorted(r["row_id"] for r in parsed)).encode()).hexdigest()
+    log_metric({"event": "data_prep", "n_rows": len(parsed), "n_a": n_a, "n_t": n_t,
+                "family_counts": fam_counts, "gold_class_counts": {f: gold_classes[f] if f == "agent_next_action_type" else {"distinct": len(gold_classes[f])} for f in FOCUS},
+                "selected_row_ids_sha256": ids_hash,
+                "seconds": round(time.time() - t0, 1)})
+    return parsed, ds, ids_hash, (n_a, n_t)
+
+
+class LazyPairDataset:
+    """Tokenize only requested pairs. Metadata stays small; no up-front Dataset.map."""
+    def __init__(self, parsed, tokenizer, tag, max_len, training_pool):
+        self.parsed, self.tokenizer, self.tag, self.max_len = parsed, tokenizer, tag, max_len
+        self.refs = []
+        for row in range(len(parsed)):
+            candidates, _ = pool_for(parsed, row, 1, training_pool)
+            self.refs.extend((row, candidate) for candidate in candidates)
+        self.metadata = Dataset.from_dict({
+            "row_idx": [r for r, _ in self.refs],
+            "cand_idx": [c for _, c in self.refs],
+            # Conservative packing bound; actual batch padding uses actual token lengths.
+            "length": [max_len] * len(self.refs),
+        })
+        self.row_sortlen = np.array([
+            min(max_len, max(1, len(r["text_a"]) // 4)) for r in parsed], dtype=np.int64)
+        log_metric({"event": "lazy_pairs_ready", "tag": tag,
+                    "n_rows": len(parsed), "n_pairs": len(self.refs),
+                    "max_len": max_len, "mapped_pairs": 0})
+
+    def __len__(self):
+        return len(self.refs)
+
+    def select_columns(self, columns):
+        return self.metadata.select_columns(columns)
+
+    def __getitem__(self, indices):
+        scalar = isinstance(indices, (int, np.integer))
+        if scalar:
+            indices = [int(indices)]
+        elif isinstance(indices, slice):
+            indices = list(range(*indices.indices(len(self))))
+        else:
+            indices = list(indices)
+        refs = [self.refs[int(i)] for i in indices]
+        encoded = self.tokenizer(
+            [self.parsed[r]["text_a"] for r, c in refs],
+            [self.parsed[r]["cand_texts"][c] for r, c in refs],
+            truncation="only_first", max_length=self.max_len,
+            padding=False, verbose=False)
+        result = {"row_idx": [r for r, c in refs],
+                  "cand_idx": [c for r, c in refs],
+                  "input_ids": encoded["input_ids"],
+                  "attention_mask": encoded["attention_mask"],
+                  "length": [len(ids) for ids in encoded["input_ids"]],
+                  "truncated": [bool(e.overflowing) for e in encoded.encodings]}
+        return {k: v[0] for k, v in result.items()} if scalar else result
+
+
+def tokenize_pairs(parsed, tokenizer, tag, max_len=MAX_LEN, num_proc=8, training_pool="full"):
+    return LazyPairDataset(parsed, tokenizer, tag, max_len, training_pool)
+
+
+class RowIndexer:
+    def __init__(self, parsed, pair_ds):
+        meta = pair_ds.select_columns(["row_idx", "cand_idx", "length"]).with_format("numpy")[:]
+        assert np.all(np.diff(meta["row_idx"]) >= 0), "pairs must be row-major"
+        self.lookup = {}
+        self.row_maxlen = np.zeros(len(parsed), dtype=np.int64)
+        for flat, (row, cand, length) in enumerate(zip(meta["row_idx"], meta["cand_idx"], meta["length"])):
+            row, cand = int(row), int(cand)
+            assert (row, cand) not in self.lookup
+            self.lookup[(row, cand)] = flat
+            self.row_maxlen[row] = max(self.row_maxlen[row], int(length))
+        assert np.all(self.row_maxlen > 0)
+        self.row_sortlen = getattr(pair_ds, "row_sortlen", self.row_maxlen)
+    def flat(self, row, cand):
+        return self.lookup[(row, cand)]
+
+
+def pool_for(parsed, row, epoch, mode):
+    """Training pool for one row: (pool_cand_indices, gold_pos_in_pool).
+    full: all declared candidates. sampled4: gold + up to 3 declared negatives,
+    deterministic per (SEED, epoch, row_id)."""
+    r = parsed[row]
+    k = len(r["cand_keys"])
+    gold = r["gold_idx"]
+    if mode == "full" or k <= 4:
+        return list(range(k)), gold
+    rng = random.Random(f"{SEED}:{epoch}:{r['row_id']}")
+    others = sorted(set(range(k)) - {gold})
+    negs = rng.sample(others, min(3, len(others)))
+    pool = [gold] + sorted(negs)
+    rng.shuffle(pool)
+    return pool, pool.index(gold)
+
+
+class TrainRefs(TorchDataset):
+    """Position i -> (row, cand_actual) for the current epoch; gold_pos is
+    recomputed in collate from the pool definition (deterministic)."""
+
+    def __init__(self, sampler):
+        self.sampler = sampler
+
+    def __len__(self):
+        return len(self.sampler.refs)
+
+    def __getitem__(self, i):
+        return self.sampler.refs[i]
+
+
+class WholeRowBatchSampler(Sampler):
+    """Yields batches of positions into self.refs. Every row's pooled pairs stay
+    in one batch (padding-token budget checked at row boundaries)."""
+
+    def __init__(self, parsed, indexer, mode, token_budget, max_rows=64,
+                 seed=SEED, shuffle=True, length_buckets=True):
+        self.parsed, self.indexer, self.mode = parsed, indexer, mode
+        self.token_budget, self.max_rows = token_budget, max_rows
+        self.rng = random.Random(seed)
+        self.shuffle = shuffle
+        self.length_buckets = length_buckets
+        self.epoch = 0
+        self.refs = []
+        self.rows_cum = 0
+        self.pairs_cum = 0
+        self.tokens_cum = 0
+        self.batches = self._build_epoch()
+
+    def _build_epoch(self):
+        self.epoch += 1
+        order = list(range(len(self.parsed)))
+        if self.shuffle:
+            self.rng.shuffle(order)
+            # Sort locally within randomized buckets to reduce padding, preserving all rows.
+            if self.length_buckets:
+                order = [row for begin in range(0, len(order), 256)
+                         for row in sorted(order[begin:begin + 256], key=lambda r: int(self.indexer.row_sortlen[r]))]
+        refs, batches, cur = [], [], []
+        batch_maxlen, batch_rows = 0, 0
+        for r in order:
+            pool, _ = pool_for(self.parsed, r, self.epoch, self.mode)
+            row_len = int(self.indexer.row_maxlen[r])
+            new_maxlen = max(batch_maxlen, row_len)
+            if cur and (new_maxlen * (len(cur) + len(pool)) > self.token_budget
+                        or batch_rows >= self.max_rows):
+                batches.append(cur)
+                cur, batch_maxlen, batch_rows = [], 0, 0
+            start = len(refs)
+            refs.extend((r, cand) for cand in pool)
+            cur.extend(range(start, start + len(pool)))
+            batch_maxlen = max(batch_maxlen, row_len)
+            batch_rows += 1
+        if cur:
+            batches.append(cur)
+        self.refs = refs
+        return batches
+
+    def __iter__(self):
+        return iter(self.batches)
+
+    def __len__(self):
+        return len(self.batches)
+
+
+def make_collate(pair_ds, indexer, pad_id, parsed, mode):
+    def collate(items):
+        rows = torch.tensor([it[0] for it in items], dtype=torch.long)
+        by_row = {}
+        for row_index, candidate_index in items:
+            by_row.setdefault(row_index, []).append(candidate_index)
+        gold_positions = {}
+        for row_index, candidate_indices in by_row.items():
+            gold_index = parsed[row_index]["gold_idx"]
+            assert candidate_indices.count(gold_index) == 1, "Each row needs exactly one gold candidate"
+            gold_positions[row_index] = candidate_indices.index(gold_index)
+        golds = torch.tensor([gold_positions[it[0]] for it in items], dtype=torch.long)
+        flats = [indexer.flat(it[0], it[1]) for it in items]
+        rec = pair_ds[flats]
+        maxlen = max(rec["length"])
+        B = len(items)
+        ids = torch.full((B, maxlen), pad_id, dtype=torch.long)
+        att = torch.zeros((B, maxlen), dtype=torch.long)
+        for i, (ii, aa) in enumerate(zip(rec["input_ids"], rec["attention_mask"])):
+            n = len(ii)
+            ids[i, :n] = torch.tensor(ii, dtype=torch.long)
+            att[i, :n] = torch.tensor(aa, dtype=torch.long)
+        return {"input_ids": ids, "attention_mask": att, "row": rows, "gold": golds,
+                "_n_truncated": sum(rec["truncated"]), "_n_at_max": sum(n >= MAX_LEN for n in rec["length"])}
+    collate.epoch = 1
+    return collate
+
+
+def grouped_ce(logits, rows, gold):
+    """Per-row softmax CE. rows must be contiguous per row (asserted)."""
+    uniq_c = torch.unique_consecutive(rows)
+    uniq_all = torch.unique(rows)
+    assert len(uniq_c) == len(uniq_all), "row pairs not contiguous — grouping unsafe"
+    counts = torch.unique_consecutive(rows, return_counts=True)[1].tolist()
+    losses = []
+    ofs = 0
+    for c in counts:
+        seg = logits[ofs:ofs + c]
+        g = int(gold[ofs].item())
+        assert g < c, f"gold {g} out of pool size {c}"
+        losses.append(F.cross_entropy(seg.unsqueeze(0),
+                                      torch.tensor([g], device=seg.device)))
+        ofs += c
+    return torch.stack(losses).mean(), len(counts)
+
+
+class GroupTrainer(Trainer):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.model_accepts_loss_kwargs = False
+
+    def compute_loss(self, model, inputs, return_outputs=False, **kwargs):
+        gold = inputs.pop("gold")
+        rows = inputs.pop("row")
+        if not hasattr(self, "seen_rows"):
+            self.seen_rows = set()
+            self.rows_processed = 0
+            self.pairs_processed = 0
+            self.tokens_processed = 0
+            self.truncated_pairs = 0
+            self.pairs_at_max = 0
+        self.truncated_pairs += int(inputs.pop("_n_truncated", 0))
+        self.pairs_at_max += int(inputs.pop("_n_at_max", 0))
+        batch_rows = torch.unique_consecutive(rows).detach().cpu().tolist()
+        self.seen_rows.update(batch_rows)
+        self.rows_processed += len(batch_rows)
+        self.pairs_processed += len(rows)
+        self.tokens_processed += int(inputs["attention_mask"].sum().item())
+        out = model(input_ids=inputs["input_ids"],
+                    attention_mask=inputs["attention_mask"])
+        logits = out.logits.squeeze(-1).float()
+        loss, _ = grouped_ce(logits, rows, gold)
+        return (loss, out) if return_outputs else loss
+
+    def get_train_dataloader(self):
+        workers = self.args.dataloader_num_workers
+        return DataLoader(self._refs_ds, batch_sampler=self._sampler,
+                          collate_fn=self._collate, num_workers=workers,
+                          persistent_workers=workers > 0,
+                          prefetch_factor=2 if workers else None,
+                          pin_memory=self.args.device.type == "cuda")
+
+
+class EvalPairs(TorchDataset):
+    def __init__(self, pair_ds):
+        self.pair_ds = pair_ds
+
+    def __len__(self):
+        return len(self.pair_ds)
+
+    def __getitem__(self, i):
+        return i
+
+
+def make_eval_collate(pair_ds, pad_id):
+    def collate(idx_list):
+        rec = pair_ds[idx_list]
+        maxlen = max(rec["length"])
+        B = len(idx_list)
+        ids = torch.full((B, maxlen), pad_id, dtype=torch.long)
+        att = torch.zeros((B, maxlen), dtype=torch.long)
+        for i, (ii, aa) in enumerate(zip(rec["input_ids"], rec["attention_mask"])):
+            n = len(ii)
+            ids[i, :n] = torch.tensor(ii, dtype=torch.long)
+            att[i, :n] = torch.tensor(aa, dtype=torch.long)
+        return {"input_ids": ids, "attention_mask": att,
+                "row": torch.tensor(rec["row_idx"], dtype=torch.long),
+                "_n_truncated": sum(rec["truncated"])}
+    return collate
+
+
+@torch.no_grad()
+def evaluate_rows(model, pair_ds, parsed, device, batch_size=8):
+    """Row-grouped accuracy over ALL declared candidates."""
+    model.eval()
+    scores = [[] for _ in range(len(parsed))]
+    n_truncated_pairs = 0
+    dl = DataLoader(EvalPairs(pair_ds), batch_size=batch_size,
+                    collate_fn=make_eval_collate(pair_ds, model.config.pad_token_id), num_workers=0)
+    t0 = time.time()
+    for batch in dl:
+        n_truncated_pairs += int(batch["_n_truncated"])
+        with torch.autocast(device_type=device, dtype=torch.bfloat16, enabled=device == "cuda"):
+            out = model(input_ids=batch["input_ids"].to(device),
+                        attention_mask=batch["attention_mask"].to(device))
+        lg = out.logits.squeeze(-1).float().cpu()
+        for j, r in enumerate(batch["row"].tolist()):
+            scores[r].append(lg[j].item())
+    correct, n = 0, 0
+    per_fam = {f: [0, 0] for f in FOCUS}
+    pred_labels = []
+    for r, row in enumerate(parsed):
+        if not scores[r]:
+            continue
+        p = min(range(len(scores[r])), key=lambda c: (-scores[r][c], row["cand_keys"][c]))
+        ok = int(p == row["gold_idx"])
+        correct += ok
+        n += 1
+        per_fam[row["family"]][0] += ok
+        per_fam[row["family"]][1] += 1
+        pred_labels.append(row["cand_keys"][p])
+    model.train()
+    return {"acc": correct / max(n, 1), "n": n,
+            "macro_accuracy": sum(per_fam[f][0] / max(per_fam[f][1], 1) for f in FOCUS) / len(FOCUS),
+            "per_family": {f: {"acc": per_fam[f][0] / max(per_fam[f][1], 1),
+                               "n": per_fam[f][1]} for f in FOCUS},
+            "pred_labels": pred_labels,
+            "n_pairs": len(pair_ds), "n_truncated_pairs": n_truncated_pairs,
+            "seconds": round(time.time() - t0, 1)}
+
+
+def stratified_subset(parsed, n, seed=SEED):
+    fam_rows = {f: sorted([r for r in parsed if r["family"] == f],
+                          key=lambda r: r["row_id"]) for f in FOCUS}
+    total = sum(len(v) for v in fam_rows.values())
+    out = []
+    for f in FOCUS:
+        k = n * len(fam_rows[f]) // total
+        out += random.Random(seed).sample(fam_rows[f], k)
+    return out[:n]
+
+
+def parse_eval_rows(split_ds):
+    parsed, skipped = [], 0
+    for r in split_ds:
+        row = parse_row(r)
+        if row["gold_idx"] is None:
+            skipped += 1
+            continue
+        parsed.append(row)
+    return parsed, skipped
+
+
+def tokenize_eval(parsed, tokenizer):
+    pd = tokenize_pairs(parsed, tokenizer, tag="eval", num_proc=4)
+    idx = RowIndexer(parsed, pd)
+    return pd, idx
+
+
+def build_baseline_results(parsed_tr, parsed_val, parsed_te):
+    """Analytic uniform expected accuracy and allowed-choice training frequency."""
+    freq = {f: {} for f in FOCUS}
+    for row in parsed_tr:
+        g = row["cand_keys"][row["gold_idx"]]
+        fam_freq = freq[row["family"]]
+        fam_freq[g] = fam_freq.get(g, 0) + 1
+    res = {}
+    for name, parsed in (("val", parsed_val), ("test", parsed_te)):
+        methods = {"uniform_expected": {f: [0., 0] for f in FOCUS},
+                   "train_frequency": {f: [0., 0] for f in FOCUS}}
+        for row in parsed:
+            fam = row["family"]
+            expected = 1. / len(row["cand_keys"])
+            best = min(row["cand_keys"], key=lambda key: (-freq[fam].get(key, 0), key))
+            matched = int(best == row["cand_keys"][row["gold_idx"]])
+            for method, value in (("uniform_expected", expected), ("train_frequency", matched)):
+                methods[method][fam][0] += value
+                methods[method][fam][1] += 1
+        res[name] = {}
+        for method, counts in methods.items():
+            per = {f: {"acc": c / n, "n": n} for f, (c, n) in counts.items()}
+            res[name][method] = {"acc": sum(c for c, _ in counts.values()) / len(parsed),
+                "n": len(parsed), "per_family": per,
+                "macro_accuracy": sum(v["acc"] for v in per.values()) / len(FOCUS)}
+    return res
+
+
+@torch.no_grad()
+def gpu_latency(model, pair_ds, device, n=100):
+    model.eval()
+    lat = []
+    for i in range(min(n + 5, len(pair_ds))):
+        rec = pair_ds[[i]]
+        ids = torch.tensor(rec["input_ids"][0], dtype=torch.long,
+                           device=device).unsqueeze(0)
+        att = torch.tensor(rec["attention_mask"][0], dtype=torch.long,
+                           device=device).unsqueeze(0)
+        if device == "cuda":
+            torch.cuda.synchronize()
+        t = time.time()
+        with torch.autocast(device_type=device, dtype=torch.bfloat16, enabled=device == "cuda"):
+            model(input_ids=ids, attention_mask=att)
+        if device == "cuda":
+            torch.cuda.synchronize()
+        if i >= 5:
+            lat.append(time.time() - t)
+    lat.sort()
+    return {"measurement": "one candidate forward pass, excludes tokenization", "warmup_pairs": 5,
+            "pair_ms_p50": round(1000 * lat[len(lat) // 2], 1),
+            "pair_ms_p95": round(1000 * lat[int(0.95 * len(lat))], 1)}
+
+
+def shuffled_invariance(model, tokenizer, parsed_subset, device, k=5, batch_size=8):
+    """Permute candidate order k times; the predicted LABEL must be identical."""
+    rng = random.Random(SEED + 1)
+    pd0, _ = tokenize_eval(parsed_subset, tokenizer)
+    base_labels = evaluate_rows(model, pd0, parsed_subset, device)["pred_labels"]
+    agree, total = 0, 0
+    for rep in range(k):
+        perm = []
+        for r in parsed_subset:
+            order = list(range(len(r["cand_texts"])))
+            rng.shuffle(order)
+            perm.append({**r, "cand_keys": [r["cand_keys"][j] for j in order],
+                         "cand_texts": [r["cand_texts"][j] for j in order],
+                         "gold_idx": order.index(r["gold_idx"])})
+        pd, _ = tokenize_eval(perm, tokenizer)
+        acc = evaluate_rows(model, pd, perm, device, batch_size)
+        agree += sum(1 for a, b in zip(base_labels, acc["pred_labels"]) if a == b)
+        total += len(base_labels)
+    return {"invariance_rate": agree / max(total, 1), "perms": k,
+            "n_rows": len(parsed_subset)}
+
+
+def parse_args():
+    p = argparse.ArgumentParser()
+    p.add_argument("--mode", choices=["pilot", "prototype"], required=True)
+    p.add_argument("--pool", choices=["full", "sampled4"], default="sampled4",
+                   help="training candidate pool (eval always uses all declared)")
+    p.add_argument("--n_rows", type=int, default=60000)
+    p.add_argument("--max_steps", type=int, default=25)
+    p.add_argument("--pilot_sampled_only", action="store_true")
+    p.add_argument("--pilot_length_buckets", action="store_true")
+    p.add_argument("--token_budget", type=int, default=32768)
+    p.add_argument("--grad_accum", type=int, default=2)
+    p.add_argument("--lr", type=float, default=2e-5)
+    p.add_argument("--eval_steps", type=int, default=400)
+    p.add_argument("--deadline_seconds", type=int, default=6600)
+    p.add_argument("--eval_reserve_seconds", type=int, default=1200)
+    p.add_argument("--save_dir", default="/output/modernjev")
+    p.add_argument("--push", action="store_true", default=False)
+    p.add_argument("--hub_model_id", default="OpenMed/ModernJEV-Decide-Preview")
+    return p.parse_args()
+
+
+def prototype_training_args(args, save_dir, device):
+    return TrainingArguments(
+        output_dir=os.path.join(save_dir, "ckpt"), per_device_train_batch_size=1,
+        gradient_accumulation_steps=args.grad_accum, learning_rate=args.lr,
+        bf16=device == "cuda", use_cpu=device == "cpu",
+        num_train_epochs=1.0, logging_steps=25, logging_first_step=True,
+        save_strategy="no", eval_strategy="no", report_to="none",
+        seed=SEED, remove_unused_columns=False,
+        lr_scheduler_type="linear", warmup_steps=0.03,
+        dataloader_num_workers=2 if device == "cuda" else 0)
+
+
+def load_model():
+    return AutoModelForSequenceClassification.from_pretrained(
+        BASE_ID, revision=BASE_REV, num_labels=1, attn_implementation=ATTN_IMPL)
+
+
+def main():
+    args = parse_args()
+    t_start = time.time()
+    torch.manual_seed(SEED)
+    random.seed(SEED)
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    gpu_name = torch.cuda.get_device_name(0) if device == "cuda" else "cpu"
+    import transformers
+    log_metric({"event": "env", "mode": args.mode, "device": device, "gpu": gpu_name,
+                "torch": torch.__version__, "transformers": transformers.__version__, "attention": ATTN_IMPL})
+    assert device == "cuda", "GPU required"
+    # Validate the complete prototype argument branch before any dataset work.
+    validated_training_args = prototype_training_args(args, args.save_dir, device) if args.mode == "prototype" else None
+    log_metric({"event": "training_api_validated", "mode": args.mode})
+
+    tokenizer = AutoTokenizer.from_pretrained(BASE_ID, revision=BASE_REV)
+    pad_id = tokenizer.pad_token_id
+    assert pad_id is not None, "tokenizer has no pad token"
+
+    n_train_rows = 500 if args.mode == "pilot" else args.n_rows
+    parsed, ds, ids_hash, (n_a, n_t) = load_and_prepare(n_train_rows)
+    if args.mode == "prototype":
+        assert ids_hash == "c5b306b0471ba104161051a7241b3bce4b69e1d959ff3e34fb06ef8eb4b077d9", "Selected subset differs from authorized frozen manifest"
+    os.makedirs(args.save_dir, exist_ok=True)
+    pair_ds = tokenize_pairs(parsed, tokenizer, tag="train",
+                             training_pool=args.pool if args.mode == "prototype" else "full")
+    indexer = RowIndexer(parsed, pair_ds)
+    collate = make_collate(pair_ds, indexer, pad_id, parsed, args.pool)
+
+    if args.mode == "pilot":
+        results = {"phases": {}}
+        pilot_modes = ("sampled4",) if args.pilot_sampled_only else ("full", "sampled4")
+        for phase_i, mode in enumerate(pilot_modes):
+            torch.manual_seed(SEED)
+            phase_steps = args.max_steps if args.pilot_sampled_only else args.max_steps // 2 + (args.max_steps % 2 if phase_i == 1 else 0)
+            model = load_model().to(device)
+            sampler = WholeRowBatchSampler(parsed, indexer, mode, args.token_budget, length_buckets=args.pilot_length_buckets)
+            targs = TrainingArguments(
+                output_dir="/tmp/pilot_" + mode, per_device_train_batch_size=1,
+                gradient_accumulation_steps=1, learning_rate=args.lr, bf16=True,
+                max_steps=phase_steps, logging_steps=3, save_strategy="no",
+                eval_strategy="no", report_to="none", seed=SEED,
+                remove_unused_columns=False)
+            trainer = GroupTrainer(model=model, args=targs,
+                                   train_dataset=TrainRefs(sampler))
+            trainer._sampler = sampler
+            trainer._collate = make_collate(pair_ds, indexer, pad_id, parsed, mode)
+            trainer._refs_ds = TrainRefs(sampler)
+            torch.cuda.reset_peak_memory_stats()
+            t0 = time.time()
+            trainer.train()
+            dt = time.time() - t0
+            results["phases"][mode] = {
+                "steps": trainer.state.global_step, "seconds": round(dt, 2),
+                "steps_per_s": round(trainer.state.global_step / dt, 3),
+                "rows_seen": len(trainer.seen_rows),
+                "rows_per_s": round(trainer.rows_processed / dt, 2),
+                "pairs_seen": trainer.pairs_processed,
+                "pairs_per_s": round(trainer.pairs_processed / dt, 1),
+                "tokens_per_s": round(trainer.tokens_processed / dt),
+                "max_mem_gb": round(torch.cuda.max_memory_allocated() / 1e9, 2)}
+            log_metric({"event": "pilot_phase", "pool": mode,
+                        **results["phases"][mode]})
+            checkpoint = os.path.join(args.save_dir, "checkpoint_" + mode)
+            model.save_pretrained(checkpoint)
+            tokenizer.save_pretrained(checkpoint)
+            del trainer
+            if mode == "full":
+                del model
+                torch.cuda.empty_cache()
+        results["latency"] = gpu_latency(model, pair_ds, device)
+        log_metric({"event": "latency_gpu", **results["latency"]})
+        results["env"] = {"gpu": gpu_name, "torch": torch.__version__}
+        os.makedirs(args.save_dir, exist_ok=True)
+        with open(os.path.join(args.save_dir, "pilot_results.json"), "w") as f:
+            json.dump(results, f, indent=2, default=str)
+        save_metrics(os.path.join(args.save_dir, "pilot_metrics.jsonl"))
+        reload_model = AutoModelForSequenceClassification.from_pretrained(os.path.join(args.save_dir, "checkpoint_sampled4"), attn_implementation=ATTN_IMPL).to(device)
+        rec = pair_ds[[0]]
+        ids = torch.tensor(rec["input_ids"], device=device)
+        att = torch.tensor(rec["attention_mask"], device=device)
+        model.eval(); reload_model.eval()
+        saved_weights = reload_model.state_dict()
+        for name, value in model.state_dict().items():
+            assert torch.equal(value, saved_weights[name]), f"Checkpoint changed parameter: {name}"
+        with torch.no_grad(), torch.autocast(device_type=device, dtype=torch.bfloat16, enabled=device == "cuda"):
+            expected_logits = model(input_ids=ids, attention_mask=att).logits.float()
+            actual_logits = reload_model(input_ids=ids, attention_mask=att).logits.float()
+        assert torch.isfinite(actual_logits).all()
+        assert torch.allclose(expected_logits, actual_logits, atol=1e-4, rtol=1e-4), "Same-precision reload mismatch"
+        results["checkpoint_parameter_equality"] = True
+        results["checkpoint_reload_verified"] = True
+        with open(os.path.join(args.save_dir, "pilot_results.json"), "w") as f:
+            json.dump(results, f, indent=2)
+        if args.push:
+            from huggingface_hub import HfApi
+            api = HfApi()
+            for filename in ("pilot_results.json", "pilot_metrics.jsonl"):
+                api.upload_file(path_or_fileobj=os.path.join(args.save_dir, filename), path_in_repo="pilot/" + filename, repo_id=args.hub_model_id, repo_type="model")
+        print("PILOT_DONE", flush=True)
+        return
+
+    # ---------------- PROTOTYPE ----------------
+    save_dir = args.save_dir
+    os.makedirs(save_dir, exist_ok=True)
+    log_metric({"event": "prototype_start", "pool": args.pool,
+                "n_rows_selected": len(parsed), "n_a": n_a, "n_t": n_t})
+
+    val_full, _ = parse_eval_rows(ds["validation"].filter(
+        lambda r: r["task_family"] in FOCUS, num_proc=8))
+    val_sub = stratified_subset(val_full, 600, seed=SEED)
+    val_pd, _ = tokenize_eval(val_sub, tokenizer)
+
+    model = load_model().to(device)
+    sampler = WholeRowBatchSampler(parsed, indexer, args.pool, args.token_budget)
+    targs = validated_training_args
+    trainer = GroupTrainer(model=model, args=targs, train_dataset=TrainRefs(sampler))
+    trainer._sampler = sampler
+    trainer._collate = collate
+    trainer._refs_ds = TrainRefs(sampler)
+
+    best = {"acc": -1.0, "step": -1}
+
+    def run_val(step):
+        acc = evaluate_rows(model, val_pd, val_sub, device)
+        acc.pop("pred_labels")
+        log_metric({"event": "val_acc", "step": step,
+                    "n_rows": len(val_sub), **acc})
+        if acc["acc"] > best["acc"]:
+            best.update(acc=acc["acc"], step=step)
+
+
+    class Guards(TrainerCallback):
+        def on_step_end(self, targs2, state, control, **kw):
+            if state.global_step <= 5 or state.global_step % 100 == 0:
+                elapsed = time.time() - t_start
+                log_metric({"event": "coverage_progress", "step": state.global_step,
+                            "rows_seen": len(trainer.seen_rows), "target": len(parsed),
+                            "elapsed_seconds": round(elapsed, 1)})
+            if state.global_step % 1000 == 0:
+                latest = os.path.join(save_dir, "latest-checkpoint")
+                model.save_pretrained(latest)
+                tokenizer.save_pretrained(latest)
+                with open(os.path.join(latest, "coverage.json"), "w") as f:
+                    json.dump({"rows_seen": len(trainer.seen_rows), "step": state.global_step}, f)
+            if state.global_step % args.eval_steps == 0 and state.global_step > 0:
+                run_val(state.global_step)
+            if time.time() - t_start > args.deadline_seconds - args.eval_reserve_seconds:
+                control.should_training_stop = True
+                log_metric({"event": "time_guard_stop", "step": state.global_step,
+                            "rows_cum": len(trainer.seen_rows)})
+
+    trainer.add_callback(Guards())
+    t0 = time.time()
+    trainer.train()
+    train_seconds = time.time() - t0
+    rows_trained = len(trainer.seen_rows)
+    steps_done = trainer.state.global_step
+    log_metric({"event": "train_done", "pool": args.pool,
+                "seconds": round(train_seconds, 1), "steps": steps_done,
+                "rows_covered": rows_trained, "n_rows_selected": len(parsed),
+                "n_pairs_processed": trainer.pairs_processed, "n_truncated_pairs": trainer.truncated_pairs,
+                "note": "rows_covered counts unique decisions actually iterated; "
+                        "no full-epoch guarantee"})
+
+    # One fixed epoch: final weights are the selected checkpoint.
+    # Validation is monitored without rewinding to a partially trained checkpoint.
+    model_dir = os.path.join(save_dir, "model")
+    model.save_pretrained(model_dir)
+    tokenizer.save_pretrained(model_dir)
+    with open(os.path.join(save_dir, "training_coverage.json"), "w") as f:
+        json.dump({"target": len(parsed), "rows_seen": rows_trained,
+                   "complete": rows_trained == len(parsed),
+                   "steps": steps_done, "selection": "final fixed-epoch checkpoint",
+                   "seen_row_ids": sorted(parsed[i]["row_id"] for i in trainer.seen_rows)}, f)
+    if args.push:
+        from huggingface_hub import HfApi
+        api = HfApi()
+        api.upload_folder(folder_path=model_dir, repo_id=args.hub_model_id, repo_type="model",
+                          commit_message="Persist final prototype before evaluation")
+        api.upload_file(path_or_fileobj=os.path.join(save_dir, "training_coverage.json"),
+                        path_in_repo="training_coverage.json", repo_id=args.hub_model_id, repo_type="model")
+    log_metric({"event": "checkpoint_saved_before_evaluation", "rows_covered": rows_trained})
+
+    results = {"model": "ModernJEV-Decide-Preview",
+               "dataset": {"id": DS_ID, "revision": DS_REV},
+               "base": {"id": BASE_ID, "revision": BASE_REV},
+               "train_pool": args.pool, "max_len": MAX_LEN, "seed": SEED,
+               "n_rows_selected": len(parsed), "n_a": n_a, "n_t": n_t,
+               "selected_row_ids_sha256": ids_hash,
+               "rows_covered": rows_trained, "steps": steps_done,
+               "train_seconds": round(train_seconds, 1),
+               "lr": args.lr, "token_budget": args.token_budget,
+               "grad_accum": args.grad_accum, "validation_monitor": best,
+               "input_preparation": "lazy per batch, no upfront map",
+               "training_input_stats": {"pairs": trainer.pairs_processed, "truncated": trainer.truncated_pairs, "at_max": trainer.pairs_at_max},
+               "checkpoint_selection": "final fixed-epoch checkpoint", "complete_training_coverage": rows_trained == len(parsed),
+               "gpu": gpu_name, "torch": torch.__version__, "attention": ATTN_IMPL,
+               "transformers": transformers.__version__}
+
+    val_pd_full, _ = tokenize_eval(val_full, tokenizer)
+    results["val_full"] = {k: v for k, v in
+                           evaluate_rows(model, val_pd_full, val_full, device).items()
+                           if k != "pred_labels"}
+    log_metric({"event": "val_full", **results["val_full"]})
+
+    test_focus, skipped_te = parse_eval_rows(ds["test"].filter(
+        lambda r: r["task_family"] in FOCUS, num_proc=8))
+    log_metric({"event": "test_prep", "n_rows": len(test_focus),
+                "skipped": skipped_te})
+    te_pd, _ = tokenize_eval(test_focus, tokenizer)
+    te = evaluate_rows(model, te_pd, test_focus, device)
+    results["test"] = {k: v for k, v in te.items() if k != "pred_labels"}
+    log_metric({"event": "final_test", **results["test"]})
+
+    inv_rows = stratified_subset(test_focus, 300, seed=SEED)
+    results["shuffled_invariance"] = shuffled_invariance(
+        model, tokenizer, inv_rows, device)
+    log_metric({"event": "shuffled_invariance", **results["shuffled_invariance"]})
+
+    results["baselines"] = build_baseline_results(parsed, val_full, test_focus)
+    log_metric({"event": "baselines", **results["baselines"]})
+
+    torch.manual_seed(SEED)
+    base_model = load_model().to(device)
+    results["baseline_untrained_head"] = {
+        "val": {k: v for k, v in evaluate_rows(
+            base_model, val_pd_full, val_full, device).items() if k != "pred_labels"},
+        "test": {k: v for k, v in evaluate_rows(
+            base_model, te_pd, test_focus, device).items() if k != "pred_labels"}}
+    log_metric({"event": "baseline_untrained_head",
+                **results["baseline_untrained_head"]})
+    del base_model
+    torch.cuda.empty_cache()
+
+    results["latency_gpu"] = gpu_latency(model, te_pd, device)
+    log_metric({"event": "latency_gpu", **results["latency_gpu"]})
+
+    results["probe"] = {"omitted": True, "reason": "Budget reserved for full prototype coverage and held-out evaluation"}
+
+    model_dir = os.path.join(save_dir, "model")
+    with open(os.path.join(save_dir, "results.json"), "w") as f:
+        json.dump(results, f, indent=2, default=str)
+    save_metrics(os.path.join(save_dir, "metrics.jsonl"))
+    with gzip.open(os.path.join(save_dir, "selected_row_ids.json.gz"), "wt") as f:
+        json.dump({"sha256": ids_hash, "n": len(parsed), "n_a": n_a, "n_t": n_t,
+                   "pool": args.pool, "rows_covered": rows_trained,
+                   "row_ids": [r["row_id"] for r in parsed]}, f)
+    here = os.path.dirname(os.path.abspath(__file__))
+    if os.path.exists(os.path.join(here, "predict.py")):
+        import shutil
+        shutil.copy(os.path.join(here, "predict.py"),
+                    os.path.join(save_dir, "predict.py"))
+    if args.push:
+        from huggingface_hub import HfApi
+        api = HfApi()
+        assert api.model_info(args.hub_model_id).private, "Private model required"
+        for name in ["results.json", "metrics.jsonl", "selected_row_ids.json.gz",
+                     "predict.py"]:
+            api.upload_file(path_or_fileobj=os.path.join(save_dir, name),
+                            repo_id=args.hub_model_id, repo_type="model",
+                            path_in_repo=name)
+    log_metric({"event": "persisted", "save_dir": save_dir, "pushed": args.push})
+    print("PROTOTYPE_DONE", flush=True)
+
+
+if __name__ == "__main__":
+    main()

results.json

@@ -0,0 +1,195 @@
+{
+  "model": "ModernJEV-Decide-Preview",
+  "dataset": {
+    "id": "MaziyarPanahi/AgentToolDecisions-180K",
+    "revision": "f2fb14e4ec977c420f376c08785664cd38763d7e"
+  },
+  "base": {
+    "id": "answerdotai/ModernBERT-base",
+    "revision": "8949b909ec900327062f0ebf497f51aef5e6f0c8"
+  },
+  "train_pool": "sampled4",
+  "max_len": 4096,
+  "seed": 42,
+  "n_rows_selected": 60000,
+  "n_a": 40809,
+  "n_t": 19191,
+  "selected_row_ids_sha256": "c5b306b0471ba104161051a7241b3bce4b69e1d959ff3e34fb06ef8eb4b077d9",
+  "rows_covered": 60000,
+  "steps": 3720,
+  "train_seconds": 7786.5,
+  "lr": 2e-05,
+  "token_budget": 114688,
+  "grad_accum": 2,
+  "validation_monitor": {
+    "acc": 0.6944908180300501,
+    "step": 3200
+  },
+  "input_preparation": "lazy per batch, no upfront map",
+  "training_input_stats": {
+    "pairs": 199191,
+    "truncated": 9249,
+    "at_max": 9265
+  },
+  "checkpoint_selection": "final fixed-epoch checkpoint",
+  "complete_training_coverage": true,
+  "gpu": "NVIDIA A100-SXM4-80GB",
+  "torch": "2.12.0+cu126",
+  "attention": "kernels-community/flash-attn2@f50dc99ed079b35990bc895d43fd353ea0cb376d",
+  "transformers": "5.17.0",
+  "val_full": {
+    "acc": 0.6840934371523916,
+    "n": 1798,
+    "macro_accuracy": 0.6529034180751321,
+    "per_family": {
+      "agent_next_action_type": {
+        "acc": 0.7348912167606769,
+        "n": 1241
+      },
+      "tool_selection": {
+        "acc": 0.5709156193895871,
+        "n": 557
+      }
+    },
+    "n_pairs": 14449,
+    "n_truncated_pairs": 443,
+    "seconds": 266.6
+  },
+  "test": {
+    "acc": 0.6982352941176471,
+    "n": 1700,
+    "macro_accuracy": 0.6778976986661058,
+    "per_family": {
+      "agent_next_action_type": {
+        "acc": 0.7340241796200345,
+        "n": 1158
+      },
+      "tool_selection": {
+        "acc": 0.6217712177121771,
+        "n": 542
+      }
+    },
+    "n_pairs": 14251,
+    "n_truncated_pairs": 414,
+    "seconds": 274.1
+  },
+  "shuffled_invariance": {
+    "invariance_rate": 1.0,
+    "perms": 5,
+    "n_rows": 299
+  },
+  "baselines": {
+    "val": {
+      "uniform_expected": {
+        "acc": 0.24665407679797663,
+        "n": 1798,
+        "per_family": {
+          "agent_next_action_type": {
+            "acc": 0.33333333333332843,
+            "n": 1241
+          },
+          "tool_selection": {
+            "acc": 0.05353207076499354,
+            "n": 557
+          }
+        },
+        "macro_accuracy": 0.193432702049161
+      },
+      "train_frequency": {
+        "acc": 0.47052280311457173,
+        "n": 1798,
+        "per_family": {
+          "agent_next_action_type": {
+            "acc": 0.5511684125705076,
+            "n": 1241
+          },
+          "tool_selection": {
+            "acc": 0.29084380610412924,
+            "n": 557
+          }
+        },
+        "macro_accuracy": 0.4210061093373184
+      }
+    },
+    "test": {
+      "uniform_expected": {
+        "acc": 0.24359277413013672,
+        "n": 1700,
+        "per_family": {
+          "agent_next_action_type": {
+            "acc": 0.33333333333332943,
+            "n": 1158
+          },
+          "tool_selection": {
+            "acc": 0.051859254651728665,
+            "n": 542
+          }
+        },
+        "macro_accuracy": 0.19259629399252903
+      },
+      "train_frequency": {
+        "acc": 0.43529411764705883,
+        "n": 1700,
+        "per_family": {
+          "agent_next_action_type": {
+            "acc": 0.531951640759931,
+            "n": 1158
+          },
+          "tool_selection": {
+            "acc": 0.22878228782287824,
+            "n": 542
+          }
+        },
+        "macro_accuracy": 0.3803669642914046
+      }
+    }
+  },
+  "baseline_untrained_head": {
+    "val": {
+      "acc": 0.2925472747497219,
+      "n": 1798,
+      "macro_accuracy": 0.21439969214610907,
+      "per_family": {
+        "agent_next_action_type": {
+          "acc": 0.41982272360999195,
+          "n": 1241
+        },
+        "tool_selection": {
+          "acc": 0.008976660682226212,
+          "n": 557
+        }
+      },
+      "n_pairs": 14449,
+      "n_truncated_pairs": 443,
+      "seconds": 267.2
+    },
+    "test": {
+      "acc": 0.29411764705882354,
+      "n": 1700,
+      "macro_accuracy": 0.2178523857777438,
+      "per_family": {
+        "agent_next_action_type": {
+          "acc": 0.4283246977547496,
+          "n": 1158
+        },
+        "tool_selection": {
+          "acc": 0.007380073800738007,
+          "n": 542
+        }
+      },
+      "n_pairs": 14251,
+      "n_truncated_pairs": 414,
+      "seconds": 274.4
+    }
+  },
+  "latency_gpu": {
+    "measurement": "one candidate forward pass, excludes tokenization",
+    "warmup_pairs": 5,
+    "pair_ms_p50": 23.6,
+    "pair_ms_p95": 24.2
+  },
+  "probe": {
+    "omitted": true,
+    "reason": "Budget reserved for full prototype coverage and held-out evaluation"
+  }
+}

runtime-versions.json

@@ -0,0 +1,8 @@
+{
+  "torch": "2.12.0+cu126",
+  "transformers": "5.17.0",
+  "datasets": "5.0.1",
+  "accelerate": "1.15.0",
+  "huggingface-hub": "1.33.0",
+  "kernels": "0.16.0"
+}

selected_row_ids.json.gz

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cd7a24a0cd78a6908610758f4ed1cc7d0cac3ca89442bdaefc3ef92c114d0f72
+size 1186946

tokenizer_config.json

@@ -0,0 +1,17 @@
+{
+  "backend": "tokenizers",
+  "clean_up_tokenization_spaces": true,
+  "cls_token": "[CLS]",
+  "is_local": false,
+  "local_files_only": false,
+  "mask_token": "[MASK]",
+  "model_input_names": [
+    "input_ids",
+    "attention_mask"
+  ],
+  "model_max_length": 8192,
+  "pad_token": "[PAD]",
+  "sep_token": "[SEP]",
+  "tokenizer_class": "TokenizersBackend",
+  "unk_token": "[UNK]"
+}

training-status.json

@@ -0,0 +1,9 @@
+{
+  "status": "training_preparation",
+  "model": "ModernJEV-Decide-Preview",
+  "training_decisions_target": 60000,
+  "max_sequence_length": 4096,
+  "compute_budget_usd": 10,
+  "weights_available": false,
+  "evaluation_available": false
+}

verification/final-coverage.json

@@ -0,0 +1,8 @@
+{
+  "target": 60000,
+  "rows_seen": 60000,
+  "complete": true,
+  "steps": 3720,
+  "selection": "final fixed-epoch checkpoint",
+  "exact_selected_row_set_verified": true
+}

verification/full-training-branch-smoke.json

@@ -0,0 +1,13 @@
+{
+  "passed": true,
+  "test": "complete one-epoch production trainer and callback path on 24 synthetic decisions",
+  "optimizer_steps": 6,
+  "rows_seen": 24,
+  "validation_calls": 3,
+  "checkpoint_parameter_equality": true,
+  "lazy_tokens_equal_to_eager": true,
+  "long_input_truncation_verified": true,
+  "warmup_steps_for_100": 3,
+  "transformers": "5.17.0",
+  "not_a_quality_evaluation": true
+}

verification/full-training-worker-smoke.json

@@ -0,0 +1,13 @@
+{
+  "passed": true,
+  "test": "complete production training path with two forked prefetch workers on 24 synthetic decisions",
+  "optimizer_steps": 6,
+  "rows_seen": 24,
+  "validation_calls": 3,
+  "checkpoint_parameter_equality": true,
+  "lazy_tokens_equal_to_eager": true,
+  "long_input_truncation_verified": true,
+  "warmup_steps_for_100": 3,
+  "transformers": "5.17.0",
+  "not_a_quality_evaluation": true
+}

verification/training-start.log

@@ -0,0 +1,170 @@
+WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.
+WARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager, possibly rendering your system unusable. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv. Use the --root-user-action option if you know what you are doing and want to suppress this warning.
+
+[notice] A new release of pip is available: 25.0.1 -> 26.2.1
+[notice] To update, run: pip install --upgrade pip
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+CUDA_KERNEL_VERIFIED {"gpu": "NVIDIA A100-SXM4-80GB", "torch": "2.12.0+cu126"}
+
+
+
+
+
+
+ACTUAL_TRAINING_LAUNCH {"n_rows": 60000, "max_len": 4096, "input_preparation": "lazy per batch; no upfront map", "remaining_seconds": 9571, "warmup_steps": 0.03}
+METRIC {"event": "env", "mode": "prototype", "device": "cuda", "gpu": "NVIDIA A100-SXM4-80GB", "torch": "2.12.0+cu126", "transformers": "5.17.0", "attention": "kernels-community/flash-attn2@f50dc99ed079b35990bc895d43fd353ea0cb376d", "ts": 1790774736.9}
+METRIC {"event": "training_api_validated", "mode": "prototype", "ts": 1790774737.1}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+METRIC {"event": "data_prep", "n_rows": 60000, "n_a": 40809, "n_t": 19191, "family_counts": {"agent_next_action_type": 40809, "tool_selection": 19191}, "gold_class_counts": {"agent_next_action_type": {"tool_call": 19173, "text_response": 21636}, "tool_selection": {"distinct": 2840}}, "selected_row_ids_sha256": "c5b306b0471ba104161051a7241b3bce4b69e1d959ff3e34fb06ef8eb4b077d9", "seconds": 25.0, "ts": 1790774762.8}
+METRIC {"event": "lazy_pairs_ready", "tag": "train", "n_rows": 60000, "n_pairs": 199191, "max_len": 4096, "mapped_pairs": 0, "ts": 1790774763.4}
+METRIC {"event": "prototype_start", "pool": "sampled4", "n_rows_selected": 60000, "n_a": 40809, "n_t": 19191, "ts": 1790774763.9}
+
+
+METRIC {"event": "lazy_pairs_ready", "tag": "eval", "n_rows": 599, "n_pairs": 4767, "max_len": 4096, "mapped_pairs": 0, "ts": 1790774765.7}
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+[transformers] ModernBertForSequenceClassification LOAD REPORT from: answerdotai/ModernBERT-base
+Key               | Status     | 
+------------------+------------+-
+decoder.bias      | UNEXPECTED | 
+classifier.bias   | MISSING    | 
+classifier.weight | MISSING    | 
+
+Notes:
+- UNEXPECTED:	can be ignored when loading from different task/architecture; not ok if you expect identical arch.
+- MISSING:	those params were newly initialized because missing from the checkpoint. Consider training on your downstream task.
+
+
  0%|          | 0/3720 [00:00<?, ?it/s]
+METRIC {"event": "coverage_progress", "step": 1, "rows_seen": 17, "target": 60000, "elapsed_seconds": 44.8, "ts": 1790774781.7}
+
  0%|          | 1/3720 [00:02<2:48:33,  2.72s/it]
+
                                                  
+{'loss': '1.181', 'grad_norm': '2.406', 'learning_rate': '0', 'epoch': '0.0002688'}
+
  0%|          | 1/3720 [00:02<2:48:33,  2.72s/it]
+METRIC {"event": "coverage_progress", "step": 2, "rows_seen": 33, "target": 60000, "elapsed_seconds": 46.2, "ts": 1790774783.0}
+
  0%|          | 2/3720 [00:04<1:57:23,  1.89s/it]
+METRIC {"event": "coverage_progress", "step": 3, "rows_seen": 49, "target": 60000, "elapsed_seconds": 47.5, "ts": 1790774784.4}
+
  0%|          | 3/3720 [00:05<1:41:43,  1.64s/it]
+METRIC {"event": "coverage_progress", "step": 4, "rows_seen": 65, "target": 60000, "elapsed_seconds": 48.9, "ts": 1790774785.7}
+
  0%|          | 4/3720 [00:06<1:35:31,  1.54s/it]
+METRIC {"event": "coverage_progress", "step": 5, "rows_seen": 81, "target": 60000, "elapsed_seconds": 50.4, "ts": 1790774787.2}

workflow/ONE-PROMPT-REPLAY.md

@@ -0,0 +1,34 @@
+# ML Intern one-prompt recipe replay: provenance
+
+## What this package contains
+
+The model weights in the parent repository are the original 60,000-decision checkpoint. This replay is a separate 6,000-decision checkpoint. None of its metrics replace the parent model's measurements.
+
+## Attempt 1 — blocked before compute
+
+Conversation: https://huggingface.co/chat/conversation/6abe15a0d2dc32fb39119ec9
+The source recipe could not be read via the HuggingChat file tools, including at its canonical private namespace. A local HF read-only check could access it. The tool failure is documented, but its platform/permission root cause is not conclusively diagnosed. ML Intern created a private provenance repository and stopped before GPU spend.
+
+## Attempt 2 — training and upload verified
+
+Conversation: https://huggingface.co/chat/conversation/6abe292848531444d725312b
+Job: https://huggingface.co/jobs/OpenMed/6abe2b6ffbc85ba6823612cc
+Destination: OpenMed/ModernJEV-Decide-OnePrompt-6K-20261001-R2 (private)
+One execution message contained the attached prompt and tested source bundle. OpenMed was selected in HuggingChat Billing; the conversation budget control was set to $10 after submission. Source preparation and those UI setup actions are disclosed. Codex performed only read-only observation/verification after submission: zero follow-up execution messages, zero code patches and zero job launches.
+
+ML Intern adapted the source, corrected its own rehearsal code, froze 6,000 rows (4,080 next action and 1,920 tool selection), performed a production-path rehearsal, reset to base for training, and launched the A100 job using its own tools. The runtime reported Torch 2.12.0+cu130, differing from the supplied cu126 reference; the pinned attention kernel ran successfully.
+
+Verified training coverage: 6,000/6,000 unique IDs; exact set equality to the frozen selection; 1,500 optimizer steps. Frozen selection hash: b1ce539158acc3f52319e843a17826021c1da30394f2632da750a8f97b6900eb.
+Checkpoint upload revision: 72cc524bfeefb2c025fc8d99dbfeafccb64259ac.
+Training elapsed (reported runtime metric): 996.9 seconds. Job setup, chat preparation and evaluation are additional.
+Rehearsal checkpoint equality passed. The first GPU run logged exact final-checkpoint reload and changes in all 136 backbone tensors. Its helper-copy finalizer then failed on undefined `__file__`, and the When2Call baseline report reversed numerator/denominator. ML Intern detected both and launched its one authorized corrective retry without follow-up instructions. The retry trained the same frozen 6,000 decisions in 1,500 optimizer steps (1006.3 seconds) and pushed checkpoint dc4067f0de30b8d46a0330d0bbaf70984577ce82. The owner subsequently narrowed the objective to one-prompt launch and real training. At 10:59 UTC Codex stopped generation and canceled the retry A100 and preparation CPU sandbox; the monitor was paused. Full replay evaluation/packaging was not completed. This supports the narrower training workflow claim; it does not satisfy the original full packaging contract.
+
+The job has a 90-minute timeout at $2.50/hour: $3.75 maximum GPU reservation for this job, not the actual cost. CPU sandbox costs, any failures and inference accounting must be recorded separately. The $6.23 remaining displayed by HuggingChat includes outstanding holds; it is not a final spend receipt. No public publication or Space change was performed.
+
+## Supported claim
+
+With a tested recipe attached and billing/budget configured, one execution message in a fresh HuggingChat ML Intern conversation led to a real 6,000-decision training run and a private checkpoint upload, without additional implementation instructions from the operator or Codex. A prior attempt failed at private-source access and is disclosed above. This is one observed workflow execution, not proof of universal reliability.
+
+## Repository packaging
+
+The owner requested a clean main-branch history for the original 60k model under MaziyarPanahi. Historical source revision identifiers above describe what was used during the experiment; after the authorized history squash, retrieve the supplied recipe-bundle.txt and verify recipe-manifest.json instead of relying on those old revisions remaining fetchable. The source contents and actual training history are preserved in the documentation.

workflow/PROMPT-R2.txt

@@ -0,0 +1,25 @@
+Run this experiment end to end in HuggingChat ML Intern. This is my single execution prompt, including authorization to launch bounded compute now. We are testing whether ML Intern can reproduce a working training recipe without further human/Codex instructions. Do not stop after a proposal. Use your own tools to prepare, train, evaluate, save, and monitor to completion. If genuinely blocked, report the exact blocker; do not pretend completion.
+
+Billing namespace: OpenMed for inference, sandboxes and EVERY Job. The HuggingChat Billing selector is already OpenMed; verify your job namespace too. New destination: OpenMed/ModernJEV-Decide-OnePrompt-6K-20261001-R2, PRIVATE model repository. Never overwrite the existing OpenMed/ModernJEV-Decide-Preview. Never create, update, deploy, or change visibility of any Space. No dashboard Spaces. Do not publish anything.
+
+Budget: maximum $10 total new Jobs/sandbox compute including preflight and failed attempts, no extensions. One A100 80GB (a100-large) at the verified current price, no H200 and no concurrent GPU jobs. Prefer one job containing rehearsal then training, timeout at most 90 minutes if price is $2.50/hour; otherwise shorten to stay under the cap. Allow at most one autonomous corrective retry only if remaining budget covers its maximum timeout. Stop any idle sandbox when done. Chat inference must also bill OpenMed; report it separately from compute if measurable.
+
+Use the previously successful private recipe as input, not its trained weights or metrics:
+source repo MaziyarPanahi/ModernJEV-Decide-Preview
+source revision 5b31f4c2778f186cda66499ec6bf1d922e00149d
+recipe/train.py, predict.py, runtime-versions.json, and available evaluation helpers/reports as format references only.
+The exact source files are supplied in the attached recipe-bundle.txt because the prior attempt demonstrated that your private-source read route returned missing despite the local HF login verifying the source exists. Use the supplied bundle as the recipe source; no private-source download is required. This source-material attachment is explicit preparation, not work you generated. You must verify your own write access to the new PRIVATE model repo before GPU spend. The bundle also includes the previous evaluate_open_labels.py runner; parameterize its old local paths to your new job and pair it with predict.py. Never copy old model metrics as new results. Do not ask for a token pasted into chat. Verify write access with a small provenance file in the new repo. Do not use external Codex or the operator's local login to run the experiment.
+
+Train a fresh ModernBERT scalar candidate ranker from answerdotai/ModernBERT-base revision 8949b909ec900327062f0ebf497f51aef5e6f0c8. Dataset: MaziyarPanahi/AgentToolDecisions-180K revision f2fb14e4ec977c420f376c08785664cd38763d7e. Exactly 6000 training decisions, task-stratified seed 42, from agent_next_action_type and tool_selection only, one epoch, 4096-token maximum. Preserve published train/validation/test splits. Freeze and hash selected row IDs before training. The original script's prototype hash assertion is for 60000 rows: parameterize it for the independently frozen 6000-row manifest and assert exact coverage, do not blindly reuse the old hash or remove verification. Use the tested sampled4 training pool, rank ALL declared candidates at evaluation. Use the fixed final-epoch checkpoint, never tune/select on test data.
+
+Known implementation requirements: one shared scalar per question/state/candidate pair, cross-entropy grouped by row_id (decision), NOT group_id (episode). Candidate text must not contain gold labels, scores, provenance or candidate indexes. Accept arbitrary caller-supplied answer names/descriptions or unique answer-string lists, not a fixed class vocabulary. Keep the source recipe's input and truncation logic. Document modifications and diff from source recipe.
+
+Known successful environment: Python 3.12, torch 2.12.0+cu126, transformers 5.17.0, datasets 5.0.1, accelerate 1.15.0, kernels 0.16.0, huggingface-hub 1.33.0. Check runtime-versions.json for the remaining pins. FlashAttention kernel reference kernels-community/flash-attn2@f50dc99ed079b35990bc895d43fd353ea0cb376d; use MODERNJEV_ATTN for the script. Kernel repo revisions differ from model repo revisions. Transformers 5.17 rejects kernels 0.17.x and removed warmup_ratio: use warmup_steps=0.03. Materialize lazy Datasets Columns before numerical array comparisons. Keep lazy batch tokenization; no ten-minute upfront full-map pass.
+
+Before loading all training candidates, instantiate the ACTUAL production TrainingArguments and run a tiny production-path rehearsal: CUDA forward/backward, optimizer update, validation, save and reload. Compare checkpoint tensors/predictions at matching precision; do not compare BF16 and FP32 with an unrealistic 1e-5 tolerance. Then reload the original base model and reset optimizer/seed for the real 6000-row run, keeping rehearsal results separate. Inspect all hardcoded 60000-row assertions, hub destinations and finalizer references before execution. Log optimizer steps, unique rows covered, elapsed time, GPU type, dependency versions, selected-row hash and checkpoint revision.
+
+Evaluate fresh results, not copied results from the source model: next action (1158 test decisions), tool choice (542), and unseen When2Call (3652) AFTER checkpoint freeze. Never train on When2Call or the three single-label task families. Reverify per-task constant-majority references 616/1158, 79/542, 1295/3652; distinguish descriptive test-majority references from allowed-choice training-frequency baselines. Retain per-task untrained ModernBERT scalar-head comparisons where feasible, clearly labeling its random head. No overall or macro accuracy in the model card or summary. Any aggregate fields from the old finalizer must be removed from presentation. Tests must not change the training recipe.
+
+Test open answers with fresh 2/3/7/20-choice inputs, reordered choices and renamed labels. Report interface acceptance separately from semantic accuracy. Save per-row predictions, per-task correct/total/baselines, training coverage, exact dependencies, executable recipe, prediction helper, and a useful model card with limitations and examples. Verify saved checkpoint reload and weight difference from the base model. Keep everything private.
+
+Proof requirement: this is ONE PROMPT WITH AN EXISTING TESTED RECIPE, not training-code invention from scratch. Record all your tool calls, job IDs/URLs, autonomous fixes/retries, cost estimates versus actual charges, source and new checkpoint revisions, and any additional authorization clicks required. Do not claim success merely because a job launched. Success means actual training of all 6000 decisions, a reloadable fresh checkpoint in the new private repo, completed per-task evaluations, and no follow-up implementation instructions from Codex or the human. If only partial success, state exactly what remains. Monitor and deliver the final result without asking me to send a second 'continue' message.

workflow/recipe-bundle.txt

@@ -0,0 +1,1143 @@
+Verified recipe source bundle. Existing trained weights and evaluation outcomes are NOT included. Source revision 5b31f4c2778f186cda66499ec6bf1d922e00149d. Extract the delimited files exactly, then document adaptations for 6000 decisions. evaluate_open_labels.py references its old local paths: parameterize dataset/model/helper paths for the new job.
+
+===== FILE: recipe/train.py =====
+"""ModernJEV-Decide-Preview — training, evaluation, persistence.
+
+Dataset : MaziyarPanahi/AgentToolDecisions-180K @ f2fb14e4ec977c420f376c08785664cd38763d7e
+Base    : answerdotai/ModernBERT-base @ 8949b909ec900327062f0ebf497f51aef5e6f0c8
+Scope   : task_family in {agent_next_action_type, tool_selection} ONLY (choice primitive).
+
+Objective: shared-encoder candidate scalar scorer; per-row_id softmax cross-entropy
+over the row's DECLARED candidates. group_id is the EPISODE, not the decision —
+grouping is ALWAYS by row_id (asserted). Candidates enter the softmax as text
+(label + criterion), so variable tool names need no fixed head and no candidate
+index is exposed to the model. Inputs contain NO gold_label / gold_json /
+gold_score / label_source / source metadata.
+
+Training pool (--pool):
+  full     — every declared candidate of the row joins the softmax.
+  sampled4 — gold + up to 3 declared negatives, deterministic per (SEED, epoch,
+             row_id). This is ordinary sampled-choice softmax within the same loss;
+             EVALUATION ALWAYS RANKS ALL DECLARED CANDIDATES regardless of --pool.
+The pilot benchmarks both objectives and the launch report states which one ran.
+
+Modes:
+  pilot     — benchmarks full vs sampled4 throughput/memory, GPU latency, save check.
+  prototype — time-guarded training on the exact stratified subset, interval monitoring
+              on a stratified subset of the OFFICIAL validation split (full-val eval
+              recorded for the final fixed-epoch checkpoint), ONE final test evaluation on all in-scope
+              test rows, baselines (uniform / train-frequency / untrained ModernBERT
+              head), shuffled-order label invariance, optional frozen-backbone probe,
+              and persistence to --save_dir (default /output/modernjev). No Hub push
+              unless --push is passed explicitly.
+
+No Space is created anywhere; metrics are metrics.jsonl + stdout only.
+"""
+import argparse
+import gzip
+import hashlib
+import json
+import os
+import random
+import time
+
+import numpy as np
+import torch
+import torch.nn.functional as F
+from datasets import Dataset, load_dataset
+from torch.utils.data import DataLoader, Dataset as TorchDataset, Sampler
+from transformers import (AutoModelForSequenceClassification, AutoTokenizer,
+                          Trainer, TrainerCallback, TrainingArguments)
+
+DS_ID = "MaziyarPanahi/AgentToolDecisions-180K"
+DS_REV = "f2fb14e4ec977c420f376c08785664cd38763d7e"
+BASE_ID = "answerdotai/ModernBERT-base"
+BASE_REV = "8949b909ec900327062f0ebf497f51aef5e6f0c8"
+FOCUS = ("agent_next_action_type", "tool_selection")
+SEED = 42
+MAX_LEN = 4096
+ATTN_IMPL = os.environ.get("MODERNJEV_ATTN", "sdpa")
+EXPECTED = {"train": 171056, "validation": 2713, "test": 6231}
+EXPECTED_FOCUS_TRAIN = 112973
+
+METRICS = []
+
+
+def log_metric(d):
+    d["ts"] = round(time.time(), 1)
+    METRICS.append(d)
+    print("METRIC " + json.dumps(d, default=str), flush=True)
+
+
+def save_metrics(path):
+    with open(path, "w") as f:
+        for d in METRICS:
+            f.write(json.dumps(d, default=str) + "\n")
+
+
+def serialize_state(row):
+    """Compact the state. Drops the policy key ONLY when it exactly equals the
+    first system message (audited dedupe rule)."""
+    state = json.loads(row["state_json"])
+    conv = state.get("conversation") or []
+    policy = state.get("policy")
+    first = conv[0] if conv else None
+    dup = (policy is not None and isinstance(first, dict)
+           and first.get("role") == "system" and first.get("content") == policy)
+    compact = {"available_tools": state.get("available_tools") or [],
+               "conversation": conv}
+    if policy is not None and not dup:
+        compact["policy"] = policy
+    return json.dumps(compact, ensure_ascii=False)
+
+
+def parse_row(r):
+    criteria = json.loads(r["criteria_json"])
+    keys = list(criteria.keys())
+    gold = r["gold_label"]
+    return {"row_id": r["row_id"], "group_id": r["group_id"],
+            "family": r["task_family"],
+            "text_a": r["question_text"] + "\n\nSTATE:\n" + serialize_state(r),
+            "cand_keys": keys,
+            "cand_texts": [f"{k}: {criteria[k]}" for k in keys],
+            "gold_idx": keys.index(gold) if gold in keys else None}
+
+
+def load_and_prepare(n_rows, seed=SEED):
+    t0 = time.time()
+    ds = load_dataset(DS_ID, revision=DS_REV)
+    for split, n in EXPECTED.items():
+        assert len(ds[split]) == n, f"{split}: {len(ds[split])} != {n}"
+    train = ds["train"].filter(lambda r: r["task_family"] in FOCUS, num_proc=8)
+    assert len(train) == EXPECTED_FOCUS_TRAIN, f"{len(train)} != {EXPECTED_FOCUS_TRAIN}"
+
+    metadata = train.select_columns(["row_id", "task_family"])[:]
+    fam_idx = {f: [] for f in FOCUS}
+    for i, fam in enumerate(metadata["task_family"]):
+        fam_idx[fam].append(i)
+    n_a = n_rows * len(fam_idx["agent_next_action_type"]) // len(train)
+    n_t = n_rows - n_a
+    rng = random.Random(seed)
+    picked = []
+    for fam, k in (("agent_next_action_type", n_a), ("tool_selection", n_t)):
+        all_ids = metadata["row_id"]
+        idxs = sorted(fam_idx[fam], key=lambda i: all_ids[i])
+        picked += rng.sample(idxs, k)
+    assert len(picked) == n_rows
+    picked.sort()
+    fields = ["row_id", "group_id", "task_family", "question_text", "state_json", "criteria_json", "gold_label"]
+    selected = train.select(picked).select_columns(fields)[:]
+    parsed = [parse_row(dict(zip(fields, values))) for values in zip(*(selected[f] for f in fields))]
+    for r in parsed:
+        assert r["gold_idx"] is not None, f"gold missing in {r['row_id']}"
+
+    fam_counts = {f: sum(1 for r in parsed if r["family"] == f) for f in FOCUS}
+    gold_classes = {f: {} for f in FOCUS}
+    for r in parsed:
+        g = r["cand_keys"][r["gold_idx"]]
+        gold_classes[r["family"]][g] = gold_classes[r["family"]].get(g, 0) + 1
+    ids_hash = hashlib.sha256(
+        "\n".join(sorted(r["row_id"] for r in parsed)).encode()).hexdigest()
+    log_metric({"event": "data_prep", "n_rows": len(parsed), "n_a": n_a, "n_t": n_t,
+                "family_counts": fam_counts, "gold_class_counts": {f: gold_classes[f] if f == "agent_next_action_type" else {"distinct": len(gold_classes[f])} for f in FOCUS},
+                "selected_row_ids_sha256": ids_hash,
+                "seconds": round(time.time() - t0, 1)})
+    return parsed, ds, ids_hash, (n_a, n_t)
+
+
+class LazyPairDataset:
+    """Tokenize only requested pairs. Metadata stays small; no up-front Dataset.map."""
+    def __init__(self, parsed, tokenizer, tag, max_len, training_pool):
+        self.parsed, self.tokenizer, self.tag, self.max_len = parsed, tokenizer, tag, max_len
+        self.refs = []
+        for row in range(len(parsed)):
+            candidates, _ = pool_for(parsed, row, 1, training_pool)
+            self.refs.extend((row, candidate) for candidate in candidates)
+        self.metadata = Dataset.from_dict({
+            "row_idx": [r for r, _ in self.refs],
+            "cand_idx": [c for _, c in self.refs],
+            # Conservative packing bound; actual batch padding uses actual token lengths.
+            "length": [max_len] * len(self.refs),
+        })
+        self.row_sortlen = np.array([
+            min(max_len, max(1, len(r["text_a"]) // 4)) for r in parsed], dtype=np.int64)
+        log_metric({"event": "lazy_pairs_ready", "tag": tag,
+                    "n_rows": len(parsed), "n_pairs": len(self.refs),
+                    "max_len": max_len, "mapped_pairs": 0})
+
+    def __len__(self):
+        return len(self.refs)
+
+    def select_columns(self, columns):
+        return self.metadata.select_columns(columns)
+
+    def __getitem__(self, indices):
+        scalar = isinstance(indices, (int, np.integer))
+        if scalar:
+            indices = [int(indices)]
+        elif isinstance(indices, slice):
+            indices = list(range(*indices.indices(len(self))))
+        else:
+            indices = list(indices)
+        refs = [self.refs[int(i)] for i in indices]
+        encoded = self.tokenizer(
+            [self.parsed[r]["text_a"] for r, c in refs],
+            [self.parsed[r]["cand_texts"][c] for r, c in refs],
+            truncation="only_first", max_length=self.max_len,
+            padding=False, verbose=False)
+        result = {"row_idx": [r for r, c in refs],
+                  "cand_idx": [c for r, c in refs],
+                  "input_ids": encoded["input_ids"],
+                  "attention_mask": encoded["attention_mask"],
+                  "length": [len(ids) for ids in encoded["input_ids"]],
+                  "truncated": [bool(e.overflowing) for e in encoded.encodings]}
+        return {k: v[0] for k, v in result.items()} if scalar else result
+
+
+def tokenize_pairs(parsed, tokenizer, tag, max_len=MAX_LEN, num_proc=8, training_pool="full"):
+    return LazyPairDataset(parsed, tokenizer, tag, max_len, training_pool)
+
+
+class RowIndexer:
+    def __init__(self, parsed, pair_ds):
+        meta = pair_ds.select_columns(["row_idx", "cand_idx", "length"]).with_format("numpy")[:]
+        assert np.all(np.diff(meta["row_idx"]) >= 0), "pairs must be row-major"
+        self.lookup = {}
+        self.row_maxlen = np.zeros(len(parsed), dtype=np.int64)
+        for flat, (row, cand, length) in enumerate(zip(meta["row_idx"], meta["cand_idx"], meta["length"])):
+            row, cand = int(row), int(cand)
+            assert (row, cand) not in self.lookup
+            self.lookup[(row, cand)] = flat
+            self.row_maxlen[row] = max(self.row_maxlen[row], int(length))
+        assert np.all(self.row_maxlen > 0)
+        self.row_sortlen = getattr(pair_ds, "row_sortlen", self.row_maxlen)
+    def flat(self, row, cand):
+        return self.lookup[(row, cand)]
+
+
+def pool_for(parsed, row, epoch, mode):
+    """Training pool for one row: (pool_cand_indices, gold_pos_in_pool).
+    full: all declared candidates. sampled4: gold + up to 3 declared negatives,
+    deterministic per (SEED, epoch, row_id)."""
+    r = parsed[row]
+    k = len(r["cand_keys"])
+    gold = r["gold_idx"]
+    if mode == "full" or k <= 4:
+        return list(range(k)), gold
+    rng = random.Random(f"{SEED}:{epoch}:{r['row_id']}")
+    others = sorted(set(range(k)) - {gold})
+    negs = rng.sample(others, min(3, len(others)))
+    pool = [gold] + sorted(negs)
+    rng.shuffle(pool)
+    return pool, pool.index(gold)
+
+
+class TrainRefs(TorchDataset):
+    """Position i -> (row, cand_actual) for the current epoch; gold_pos is
+    recomputed in collate from the pool definition (deterministic)."""
+
+    def __init__(self, sampler):
+        self.sampler = sampler
+
+    def __len__(self):
+        return len(self.sampler.refs)
+
+    def __getitem__(self, i):
+        return self.sampler.refs[i]
+
+
+class WholeRowBatchSampler(Sampler):
+    """Yields batches of positions into self.refs. Every row's pooled pairs stay
+    in one batch (padding-token budget checked at row boundaries)."""
+
+    def __init__(self, parsed, indexer, mode, token_budget, max_rows=64,
+                 seed=SEED, shuffle=True, length_buckets=True):
+        self.parsed, self.indexer, self.mode = parsed, indexer, mode
+        self.token_budget, self.max_rows = token_budget, max_rows
+        self.rng = random.Random(seed)
+        self.shuffle = shuffle
+        self.length_buckets = length_buckets
+        self.epoch = 0
+        self.refs = []
+        self.rows_cum = 0
+        self.pairs_cum = 0
+        self.tokens_cum = 0
+        self.batches = self._build_epoch()
+
+    def _build_epoch(self):
+        self.epoch += 1
+        order = list(range(len(self.parsed)))
+        if self.shuffle:
+            self.rng.shuffle(order)
+            # Sort locally within randomized buckets to reduce padding, preserving all rows.
+            if self.length_buckets:
+                order = [row for begin in range(0, len(order), 256)
+                         for row in sorted(order[begin:begin + 256], key=lambda r: int(self.indexer.row_sortlen[r]))]
+        refs, batches, cur = [], [], []
+        batch_maxlen, batch_rows = 0, 0
+        for r in order:
+            pool, _ = pool_for(self.parsed, r, self.epoch, self.mode)
+            row_len = int(self.indexer.row_maxlen[r])
+            new_maxlen = max(batch_maxlen, row_len)
+            if cur and (new_maxlen * (len(cur) + len(pool)) > self.token_budget
+                        or batch_rows >= self.max_rows):
+                batches.append(cur)
+                cur, batch_maxlen, batch_rows = [], 0, 0
+            start = len(refs)
+            refs.extend((r, cand) for cand in pool)
+            cur.extend(range(start, start + len(pool)))
+            batch_maxlen = max(batch_maxlen, row_len)
+            batch_rows += 1
+        if cur:
+            batches.append(cur)
+        self.refs = refs
+        return batches
+
+    def __iter__(self):
+        return iter(self.batches)
+
+    def __len__(self):
+        return len(self.batches)
+
+
+def make_collate(pair_ds, indexer, pad_id, parsed, mode):
+    def collate(items):
+        rows = torch.tensor([it[0] for it in items], dtype=torch.long)
+        by_row = {}
+        for row_index, candidate_index in items:
+            by_row.setdefault(row_index, []).append(candidate_index)
+        gold_positions = {}
+        for row_index, candidate_indices in by_row.items():
+            gold_index = parsed[row_index]["gold_idx"]
+            assert candidate_indices.count(gold_index) == 1, "Each row needs exactly one gold candidate"
+            gold_positions[row_index] = candidate_indices.index(gold_index)
+        golds = torch.tensor([gold_positions[it[0]] for it in items], dtype=torch.long)
+        flats = [indexer.flat(it[0], it[1]) for it in items]
+        rec = pair_ds[flats]
+        maxlen = max(rec["length"])
+        B = len(items)
+        ids = torch.full((B, maxlen), pad_id, dtype=torch.long)
+        att = torch.zeros((B, maxlen), dtype=torch.long)
+        for i, (ii, aa) in enumerate(zip(rec["input_ids"], rec["attention_mask"])):
+            n = len(ii)
+            ids[i, :n] = torch.tensor(ii, dtype=torch.long)
+            att[i, :n] = torch.tensor(aa, dtype=torch.long)
+        return {"input_ids": ids, "attention_mask": att, "row": rows, "gold": golds,
+                "_n_truncated": sum(rec["truncated"]), "_n_at_max": sum(n >= MAX_LEN for n in rec["length"])}
+    collate.epoch = 1
+    return collate
+
+
+def grouped_ce(logits, rows, gold):
+    """Per-row softmax CE. rows must be contiguous per row (asserted)."""
+    uniq_c = torch.unique_consecutive(rows)
+    uniq_all = torch.unique(rows)
+    assert len(uniq_c) == len(uniq_all), "row pairs not contiguous — grouping unsafe"
+    counts = torch.unique_consecutive(rows, return_counts=True)[1].tolist()
+    losses = []
+    ofs = 0
+    for c in counts:
+        seg = logits[ofs:ofs + c]
+        g = int(gold[ofs].item())
+        assert g < c, f"gold {g} out of pool size {c}"
+        losses.append(F.cross_entropy(seg.unsqueeze(0),
+                                      torch.tensor([g], device=seg.device)))
+        ofs += c
+    return torch.stack(losses).mean(), len(counts)
+
+
+class GroupTrainer(Trainer):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.model_accepts_loss_kwargs = False
+
+    def compute_loss(self, model, inputs, return_outputs=False, **kwargs):
+        gold = inputs.pop("gold")
+        rows = inputs.pop("row")
+        if not hasattr(self, "seen_rows"):
+            self.seen_rows = set()
+            self.rows_processed = 0
+            self.pairs_processed = 0
+            self.tokens_processed = 0
+            self.truncated_pairs = 0
+            self.pairs_at_max = 0
+        self.truncated_pairs += int(inputs.pop("_n_truncated", 0))
+        self.pairs_at_max += int(inputs.pop("_n_at_max", 0))
+        batch_rows = torch.unique_consecutive(rows).detach().cpu().tolist()
+        self.seen_rows.update(batch_rows)
+        self.rows_processed += len(batch_rows)
+        self.pairs_processed += len(rows)
+        self.tokens_processed += int(inputs["attention_mask"].sum().item())
+        out = model(input_ids=inputs["input_ids"],
+                    attention_mask=inputs["attention_mask"])
+        logits = out.logits.squeeze(-1).float()
+        loss, _ = grouped_ce(logits, rows, gold)
+        return (loss, out) if return_outputs else loss
+
+    def get_train_dataloader(self):
+        workers = self.args.dataloader_num_workers
+        return DataLoader(self._refs_ds, batch_sampler=self._sampler,
+                          collate_fn=self._collate, num_workers=workers,
+                          persistent_workers=workers > 0,
+                          prefetch_factor=2 if workers else None,
+                          pin_memory=self.args.device.type == "cuda")
+
+
+class EvalPairs(TorchDataset):
+    def __init__(self, pair_ds):
+        self.pair_ds = pair_ds
+
+    def __len__(self):
+        return len(self.pair_ds)
+
+    def __getitem__(self, i):
+        return i
+
+
+def make_eval_collate(pair_ds, pad_id):
+    def collate(idx_list):
+        rec = pair_ds[idx_list]
+        maxlen = max(rec["length"])
+        B = len(idx_list)
+        ids = torch.full((B, maxlen), pad_id, dtype=torch.long)
+        att = torch.zeros((B, maxlen), dtype=torch.long)
+        for i, (ii, aa) in enumerate(zip(rec["input_ids"], rec["attention_mask"])):
+            n = len(ii)
+            ids[i, :n] = torch.tensor(ii, dtype=torch.long)
+            att[i, :n] = torch.tensor(aa, dtype=torch.long)
+        return {"input_ids": ids, "attention_mask": att,
+                "row": torch.tensor(rec["row_idx"], dtype=torch.long),
+                "_n_truncated": sum(rec["truncated"])}
+    return collate
+
+
+@torch.no_grad()
+def evaluate_rows(model, pair_ds, parsed, device, batch_size=8):
+    """Row-grouped accuracy over ALL declared candidates."""
+    model.eval()
+    scores = [[] for _ in range(len(parsed))]
+    n_truncated_pairs = 0
+    dl = DataLoader(EvalPairs(pair_ds), batch_size=batch_size,
+                    collate_fn=make_eval_collate(pair_ds, model.config.pad_token_id), num_workers=0)
+    t0 = time.time()
+    for batch in dl:
+        n_truncated_pairs += int(batch["_n_truncated"])
+        with torch.autocast(device_type=device, dtype=torch.bfloat16, enabled=device == "cuda"):
+            out = model(input_ids=batch["input_ids"].to(device),
+                        attention_mask=batch["attention_mask"].to(device))
+        lg = out.logits.squeeze(-1).float().cpu()
+        for j, r in enumerate(batch["row"].tolist()):
+            scores[r].append(lg[j].item())
+    correct, n = 0, 0
+    per_fam = {f: [0, 0] for f in FOCUS}
+    pred_labels = []
+    for r, row in enumerate(parsed):
+        if not scores[r]:
+            continue
+        p = min(range(len(scores[r])), key=lambda c: (-scores[r][c], row["cand_keys"][c]))
+        ok = int(p == row["gold_idx"])
+        correct += ok
+        n += 1
+        per_fam[row["family"]][0] += ok
+        per_fam[row["family"]][1] += 1
+        pred_labels.append(row["cand_keys"][p])
+    model.train()
+    return {"acc": correct / max(n, 1), "n": n,
+            "macro_accuracy": sum(per_fam[f][0] / max(per_fam[f][1], 1) for f in FOCUS) / len(FOCUS),
+            "per_family": {f: {"acc": per_fam[f][0] / max(per_fam[f][1], 1),
+                               "n": per_fam[f][1]} for f in FOCUS},
+            "pred_labels": pred_labels,
+            "n_pairs": len(pair_ds), "n_truncated_pairs": n_truncated_pairs,
+            "seconds": round(time.time() - t0, 1)}
+
+
+def stratified_subset(parsed, n, seed=SEED):
+    fam_rows = {f: sorted([r for r in parsed if r["family"] == f],
+                          key=lambda r: r["row_id"]) for f in FOCUS}
+    total = sum(len(v) for v in fam_rows.values())
+    out = []
+    for f in FOCUS:
+        k = n * len(fam_rows[f]) // total
+        out += random.Random(seed).sample(fam_rows[f], k)
+    return out[:n]
+
+
+def parse_eval_rows(split_ds):
+    parsed, skipped = [], 0
+    for r in split_ds:
+        row = parse_row(r)
+        if row["gold_idx"] is None:
+            skipped += 1
+            continue
+        parsed.append(row)
+    return parsed, skipped
+
+
+def tokenize_eval(parsed, tokenizer):
+    pd = tokenize_pairs(parsed, tokenizer, tag="eval", num_proc=4)
+    idx = RowIndexer(parsed, pd)
+    return pd, idx
+
+
+def build_baseline_results(parsed_tr, parsed_val, parsed_te):
+    """Analytic uniform expected accuracy and allowed-choice training frequency."""
+    freq = {f: {} for f in FOCUS}
+    for row in parsed_tr:
+        g = row["cand_keys"][row["gold_idx"]]
+        fam_freq = freq[row["family"]]
+        fam_freq[g] = fam_freq.get(g, 0) + 1
+    res = {}
+    for name, parsed in (("val", parsed_val), ("test", parsed_te)):
+        methods = {"uniform_expected": {f: [0., 0] for f in FOCUS},
+                   "train_frequency": {f: [0., 0] for f in FOCUS}}
+        for row in parsed:
+            fam = row["family"]
+            expected = 1. / len(row["cand_keys"])
+            best = min(row["cand_keys"], key=lambda key: (-freq[fam].get(key, 0), key))
+            matched = int(best == row["cand_keys"][row["gold_idx"]])
+            for method, value in (("uniform_expected", expected), ("train_frequency", matched)):
+                methods[method][fam][0] += value
+                methods[method][fam][1] += 1
+        res[name] = {}
+        for method, counts in methods.items():
+            per = {f: {"acc": c / n, "n": n} for f, (c, n) in counts.items()}
+            res[name][method] = {"acc": sum(c for c, _ in counts.values()) / len(parsed),
+                "n": len(parsed), "per_family": per,
+                "macro_accuracy": sum(v["acc"] for v in per.values()) / len(FOCUS)}
+    return res
+
+
+@torch.no_grad()
+def gpu_latency(model, pair_ds, device, n=100):
+    model.eval()
+    lat = []
+    for i in range(min(n + 5, len(pair_ds))):
+        rec = pair_ds[[i]]
+        ids = torch.tensor(rec["input_ids"][0], dtype=torch.long,
+                           device=device).unsqueeze(0)
+        att = torch.tensor(rec["attention_mask"][0], dtype=torch.long,
+                           device=device).unsqueeze(0)
+        if device == "cuda":
+            torch.cuda.synchronize()
+        t = time.time()
+        with torch.autocast(device_type=device, dtype=torch.bfloat16, enabled=device == "cuda"):
+            model(input_ids=ids, attention_mask=att)
+        if device == "cuda":
+            torch.cuda.synchronize()
+        if i >= 5:
+            lat.append(time.time() - t)
+    lat.sort()
+    return {"measurement": "one candidate forward pass, excludes tokenization", "warmup_pairs": 5,
+            "pair_ms_p50": round(1000 * lat[len(lat) // 2], 1),
+            "pair_ms_p95": round(1000 * lat[int(0.95 * len(lat))], 1)}
+
+
+def shuffled_invariance(model, tokenizer, parsed_subset, device, k=5, batch_size=8):
+    """Permute candidate order k times; the predicted LABEL must be identical."""
+    rng = random.Random(SEED + 1)
+    pd0, _ = tokenize_eval(parsed_subset, tokenizer)
+    base_labels = evaluate_rows(model, pd0, parsed_subset, device)["pred_labels"]
+    agree, total = 0, 0
+    for rep in range(k):
+        perm = []
+        for r in parsed_subset:
+            order = list(range(len(r["cand_texts"])))
+            rng.shuffle(order)
+            perm.append({**r, "cand_keys": [r["cand_keys"][j] for j in order],
+                         "cand_texts": [r["cand_texts"][j] for j in order],
+                         "gold_idx": order.index(r["gold_idx"])})
+        pd, _ = tokenize_eval(perm, tokenizer)
+        acc = evaluate_rows(model, pd, perm, device, batch_size)
+        agree += sum(1 for a, b in zip(base_labels, acc["pred_labels"]) if a == b)
+        total += len(base_labels)
+    return {"invariance_rate": agree / max(total, 1), "perms": k,
+            "n_rows": len(parsed_subset)}
+
+
+def parse_args():
+    p = argparse.ArgumentParser()
+    p.add_argument("--mode", choices=["pilot", "prototype"], required=True)
+    p.add_argument("--pool", choices=["full", "sampled4"], default="sampled4",
+                   help="training candidate pool (eval always uses all declared)")
+    p.add_argument("--n_rows", type=int, default=60000)
+    p.add_argument("--max_steps", type=int, default=25)
+    p.add_argument("--pilot_sampled_only", action="store_true")
+    p.add_argument("--pilot_length_buckets", action="store_true")
+    p.add_argument("--token_budget", type=int, default=32768)
+    p.add_argument("--grad_accum", type=int, default=2)
+    p.add_argument("--lr", type=float, default=2e-5)
+    p.add_argument("--eval_steps", type=int, default=400)
+    p.add_argument("--deadline_seconds", type=int, default=6600)
+    p.add_argument("--eval_reserve_seconds", type=int, default=1200)
+    p.add_argument("--save_dir", default="/output/modernjev")
+    p.add_argument("--push", action="store_true", default=False)
+    p.add_argument("--hub_model_id", default="OpenMed/ModernJEV-Decide-Preview")
+    return p.parse_args()
+
+
+def prototype_training_args(args, save_dir, device):
+    return TrainingArguments(
+        output_dir=os.path.join(save_dir, "ckpt"), per_device_train_batch_size=1,
+        gradient_accumulation_steps=args.grad_accum, learning_rate=args.lr,
+        bf16=device == "cuda", use_cpu=device == "cpu",
+        num_train_epochs=1.0, logging_steps=25, logging_first_step=True,
+        save_strategy="no", eval_strategy="no", report_to="none",
+        seed=SEED, remove_unused_columns=False,
+        lr_scheduler_type="linear", warmup_steps=0.03,
+        dataloader_num_workers=2 if device == "cuda" else 0)
+
+
+def load_model():
+    return AutoModelForSequenceClassification.from_pretrained(
+        BASE_ID, revision=BASE_REV, num_labels=1, attn_implementation=ATTN_IMPL)
+
+
+def main():
+    args = parse_args()
+    t_start = time.time()
+    torch.manual_seed(SEED)
+    random.seed(SEED)
+    device = "cuda" if torch.cuda.is_available() else "cpu"
+    gpu_name = torch.cuda.get_device_name(0) if device == "cuda" else "cpu"
+    import transformers
+    log_metric({"event": "env", "mode": args.mode, "device": device, "gpu": gpu_name,
+                "torch": torch.__version__, "transformers": transformers.__version__, "attention": ATTN_IMPL})
+    assert device == "cuda", "GPU required"
+    # Validate the complete prototype argument branch before any dataset work.
+    validated_training_args = prototype_training_args(args, args.save_dir, device) if args.mode == "prototype" else None
+    log_metric({"event": "training_api_validated", "mode": args.mode})
+
+    tokenizer = AutoTokenizer.from_pretrained(BASE_ID, revision=BASE_REV)
+    pad_id = tokenizer.pad_token_id
+    assert pad_id is not None, "tokenizer has no pad token"
+
+    n_train_rows = 500 if args.mode == "pilot" else args.n_rows
+    parsed, ds, ids_hash, (n_a, n_t) = load_and_prepare(n_train_rows)
+    if args.mode == "prototype":
+        assert ids_hash == "c5b306b0471ba104161051a7241b3bce4b69e1d959ff3e34fb06ef8eb4b077d9", "Selected subset differs from authorized frozen manifest"
+    os.makedirs(args.save_dir, exist_ok=True)
+    pair_ds = tokenize_pairs(parsed, tokenizer, tag="train",
+                             training_pool=args.pool if args.mode == "prototype" else "full")
+    indexer = RowIndexer(parsed, pair_ds)
+    collate = make_collate(pair_ds, indexer, pad_id, parsed, args.pool)
+
+    if args.mode == "pilot":
+        results = {"phases": {}}
+        pilot_modes = ("sampled4",) if args.pilot_sampled_only else ("full", "sampled4")
+        for phase_i, mode in enumerate(pilot_modes):
+            torch.manual_seed(SEED)
+            phase_steps = args.max_steps if args.pilot_sampled_only else args.max_steps // 2 + (args.max_steps % 2 if phase_i == 1 else 0)
+            model = load_model().to(device)
+            sampler = WholeRowBatchSampler(parsed, indexer, mode, args.token_budget, length_buckets=args.pilot_length_buckets)
+            targs = TrainingArguments(
+                output_dir="/tmp/pilot_" + mode, per_device_train_batch_size=1,
+                gradient_accumulation_steps=1, learning_rate=args.lr, bf16=True,
+                max_steps=phase_steps, logging_steps=3, save_strategy="no",
+                eval_strategy="no", report_to="none", seed=SEED,
+                remove_unused_columns=False)
+            trainer = GroupTrainer(model=model, args=targs,
+                                   train_dataset=TrainRefs(sampler))
+            trainer._sampler = sampler
+            trainer._collate = make_collate(pair_ds, indexer, pad_id, parsed, mode)
+            trainer._refs_ds = TrainRefs(sampler)
+            torch.cuda.reset_peak_memory_stats()
+            t0 = time.time()
+            trainer.train()
+            dt = time.time() - t0
+            results["phases"][mode] = {
+                "steps": trainer.state.global_step, "seconds": round(dt, 2),
+                "steps_per_s": round(trainer.state.global_step / dt, 3),
+                "rows_seen": len(trainer.seen_rows),
+                "rows_per_s": round(trainer.rows_processed / dt, 2),
+                "pairs_seen": trainer.pairs_processed,
+                "pairs_per_s": round(trainer.pairs_processed / dt, 1),
+                "tokens_per_s": round(trainer.tokens_processed / dt),
+                "max_mem_gb": round(torch.cuda.max_memory_allocated() / 1e9, 2)}
+            log_metric({"event": "pilot_phase", "pool": mode,
+                        **results["phases"][mode]})
+            checkpoint = os.path.join(args.save_dir, "checkpoint_" + mode)
+            model.save_pretrained(checkpoint)
+            tokenizer.save_pretrained(checkpoint)
+            del trainer
+            if mode == "full":
+                del model
+                torch.cuda.empty_cache()
+        results["latency"] = gpu_latency(model, pair_ds, device)
+        log_metric({"event": "latency_gpu", **results["latency"]})
+        results["env"] = {"gpu": gpu_name, "torch": torch.__version__}
+        os.makedirs(args.save_dir, exist_ok=True)
+        with open(os.path.join(args.save_dir, "pilot_results.json"), "w") as f:
+            json.dump(results, f, indent=2, default=str)
+        save_metrics(os.path.join(args.save_dir, "pilot_metrics.jsonl"))
+        reload_model = AutoModelForSequenceClassification.from_pretrained(os.path.join(args.save_dir, "checkpoint_sampled4"), attn_implementation=ATTN_IMPL).to(device)
+        rec = pair_ds[[0]]
+        ids = torch.tensor(rec["input_ids"], device=device)
+        att = torch.tensor(rec["attention_mask"], device=device)
+        model.eval(); reload_model.eval()
+        saved_weights = reload_model.state_dict()
+        for name, value in model.state_dict().items():
+            assert torch.equal(value, saved_weights[name]), f"Checkpoint changed parameter: {name}"
+        with torch.no_grad(), torch.autocast(device_type=device, dtype=torch.bfloat16, enabled=device == "cuda"):
+            expected_logits = model(input_ids=ids, attention_mask=att).logits.float()
+            actual_logits = reload_model(input_ids=ids, attention_mask=att).logits.float()
+        assert torch.isfinite(actual_logits).all()
+        assert torch.allclose(expected_logits, actual_logits, atol=1e-4, rtol=1e-4), "Same-precision reload mismatch"
+        results["checkpoint_parameter_equality"] = True
+        results["checkpoint_reload_verified"] = True
+        with open(os.path.join(args.save_dir, "pilot_results.json"), "w") as f:
+            json.dump(results, f, indent=2)
+        if args.push:
+            from huggingface_hub import HfApi
+            api = HfApi()
+            for filename in ("pilot_results.json", "pilot_metrics.jsonl"):
+                api.upload_file(path_or_fileobj=os.path.join(args.save_dir, filename), path_in_repo="pilot/" + filename, repo_id=args.hub_model_id, repo_type="model")
+        print("PILOT_DONE", flush=True)
+        return
+
+    # ---------------- PROTOTYPE ----------------
+    save_dir = args.save_dir
+    os.makedirs(save_dir, exist_ok=True)
+    log_metric({"event": "prototype_start", "pool": args.pool,
+                "n_rows_selected": len(parsed), "n_a": n_a, "n_t": n_t})
+
+    val_full, _ = parse_eval_rows(ds["validation"].filter(
+        lambda r: r["task_family"] in FOCUS, num_proc=8))
+    val_sub = stratified_subset(val_full, 600, seed=SEED)
+    val_pd, _ = tokenize_eval(val_sub, tokenizer)
+
+    model = load_model().to(device)
+    sampler = WholeRowBatchSampler(parsed, indexer, args.pool, args.token_budget)
+    targs = validated_training_args
+    trainer = GroupTrainer(model=model, args=targs, train_dataset=TrainRefs(sampler))
+    trainer._sampler = sampler
+    trainer._collate = collate
+    trainer._refs_ds = TrainRefs(sampler)
+
+    best = {"acc": -1.0, "step": -1}
+
+    def run_val(step):
+        acc = evaluate_rows(model, val_pd, val_sub, device)
+        acc.pop("pred_labels")
+        log_metric({"event": "val_acc", "step": step,
+                    "n_rows": len(val_sub), **acc})
+        if acc["acc"] > best["acc"]:
+            best.update(acc=acc["acc"], step=step)
+
+
+    class Guards(TrainerCallback):
+        def on_step_end(self, targs2, state, control, **kw):
+            if state.global_step <= 5 or state.global_step % 100 == 0:
+                elapsed = time.time() - t_start
+                log_metric({"event": "coverage_progress", "step": state.global_step,
+                            "rows_seen": len(trainer.seen_rows), "target": len(parsed),
+                            "elapsed_seconds": round(elapsed, 1)})
+            if state.global_step % 1000 == 0:
+                latest = os.path.join(save_dir, "latest-checkpoint")
+                model.save_pretrained(latest)
+                tokenizer.save_pretrained(latest)
+                with open(os.path.join(latest, "coverage.json"), "w") as f:
+                    json.dump({"rows_seen": len(trainer.seen_rows), "step": state.global_step}, f)
+            if state.global_step % args.eval_steps == 0 and state.global_step > 0:
+                run_val(state.global_step)
+            if time.time() - t_start > args.deadline_seconds - args.eval_reserve_seconds:
+                control.should_training_stop = True
+                log_metric({"event": "time_guard_stop", "step": state.global_step,
+                            "rows_cum": len(trainer.seen_rows)})
+
+    trainer.add_callback(Guards())
+    t0 = time.time()
+    trainer.train()
+    train_seconds = time.time() - t0
+    rows_trained = len(trainer.seen_rows)
+    steps_done = trainer.state.global_step
+    log_metric({"event": "train_done", "pool": args.pool,
+                "seconds": round(train_seconds, 1), "steps": steps_done,
+                "rows_covered": rows_trained, "n_rows_selected": len(parsed),
+                "n_pairs_processed": trainer.pairs_processed, "n_truncated_pairs": trainer.truncated_pairs,
+                "note": "rows_covered counts unique decisions actually iterated; "
+                        "no full-epoch guarantee"})
+
+    # One fixed epoch: final weights are the selected checkpoint.
+    # Validation is monitored without rewinding to a partially trained checkpoint.
+    model_dir = os.path.join(save_dir, "model")
+    model.save_pretrained(model_dir)
+    tokenizer.save_pretrained(model_dir)
+    with open(os.path.join(save_dir, "training_coverage.json"), "w") as f:
+        json.dump({"target": len(parsed), "rows_seen": rows_trained,
+                   "complete": rows_trained == len(parsed),
+                   "steps": steps_done, "selection": "final fixed-epoch checkpoint",
+                   "seen_row_ids": sorted(parsed[i]["row_id"] for i in trainer.seen_rows)}, f)
+    if args.push:
+        from huggingface_hub import HfApi
+        api = HfApi()
+        api.upload_folder(folder_path=model_dir, repo_id=args.hub_model_id, repo_type="model",
+                          commit_message="Persist final prototype before evaluation")
+        api.upload_file(path_or_fileobj=os.path.join(save_dir, "training_coverage.json"),
+                        path_in_repo="training_coverage.json", repo_id=args.hub_model_id, repo_type="model")
+    log_metric({"event": "checkpoint_saved_before_evaluation", "rows_covered": rows_trained})
+
+    results = {"model": "ModernJEV-Decide-Preview",
+               "dataset": {"id": DS_ID, "revision": DS_REV},
+               "base": {"id": BASE_ID, "revision": BASE_REV},
+               "train_pool": args.pool, "max_len": MAX_LEN, "seed": SEED,
+               "n_rows_selected": len(parsed), "n_a": n_a, "n_t": n_t,
+               "selected_row_ids_sha256": ids_hash,
+               "rows_covered": rows_trained, "steps": steps_done,
+               "train_seconds": round(train_seconds, 1),
+               "lr": args.lr, "token_budget": args.token_budget,
+               "grad_accum": args.grad_accum, "validation_monitor": best,
+               "input_preparation": "lazy per batch, no upfront map",
+               "training_input_stats": {"pairs": trainer.pairs_processed, "truncated": trainer.truncated_pairs, "at_max": trainer.pairs_at_max},
+               "checkpoint_selection": "final fixed-epoch checkpoint", "complete_training_coverage": rows_trained == len(parsed),
+               "gpu": gpu_name, "torch": torch.__version__, "attention": ATTN_IMPL,
+               "transformers": transformers.__version__}
+
+    val_pd_full, _ = tokenize_eval(val_full, tokenizer)
+    results["val_full"] = {k: v for k, v in
+                           evaluate_rows(model, val_pd_full, val_full, device).items()
+                           if k != "pred_labels"}
+    log_metric({"event": "val_full", **results["val_full"]})
+
+    test_focus, skipped_te = parse_eval_rows(ds["test"].filter(
+        lambda r: r["task_family"] in FOCUS, num_proc=8))
+    log_metric({"event": "test_prep", "n_rows": len(test_focus),
+                "skipped": skipped_te})
+    te_pd, _ = tokenize_eval(test_focus, tokenizer)
+    te = evaluate_rows(model, te_pd, test_focus, device)
+    results["test"] = {k: v for k, v in te.items() if k != "pred_labels"}
+    log_metric({"event": "final_test", **results["test"]})
+
+    inv_rows = stratified_subset(test_focus, 300, seed=SEED)
+    results["shuffled_invariance"] = shuffled_invariance(
+        model, tokenizer, inv_rows, device)
+    log_metric({"event": "shuffled_invariance", **results["shuffled_invariance"]})
+
+    results["baselines"] = build_baseline_results(parsed, val_full, test_focus)
+    log_metric({"event": "baselines", **results["baselines"]})
+
+    torch.manual_seed(SEED)
+    base_model = load_model().to(device)
+    results["baseline_untrained_head"] = {
+        "val": {k: v for k, v in evaluate_rows(
+            base_model, val_pd_full, val_full, device).items() if k != "pred_labels"},
+        "test": {k: v for k, v in evaluate_rows(
+            base_model, te_pd, test_focus, device).items() if k != "pred_labels"}}
+    log_metric({"event": "baseline_untrained_head",
+                **results["baseline_untrained_head"]})
+    del base_model
+    torch.cuda.empty_cache()
+
+    results["latency_gpu"] = gpu_latency(model, te_pd, device)
+    log_metric({"event": "latency_gpu", **results["latency_gpu"]})
+
+    results["probe"] = {"omitted": True, "reason": "Budget reserved for full prototype coverage and held-out evaluation"}
+
+    model_dir = os.path.join(save_dir, "model")
+    with open(os.path.join(save_dir, "results.json"), "w") as f:
+        json.dump(results, f, indent=2, default=str)
+    save_metrics(os.path.join(save_dir, "metrics.jsonl"))
+    with gzip.open(os.path.join(save_dir, "selected_row_ids.json.gz"), "wt") as f:
+        json.dump({"sha256": ids_hash, "n": len(parsed), "n_a": n_a, "n_t": n_t,
+                   "pool": args.pool, "rows_covered": rows_trained,
+                   "row_ids": [r["row_id"] for r in parsed]}, f)
+    here = os.path.dirname(os.path.abspath(__file__))
+    if os.path.exists(os.path.join(here, "predict.py")):
+        import shutil
+        shutil.copy(os.path.join(here, "predict.py"),
+                    os.path.join(save_dir, "predict.py"))
+    if args.push:
+        from huggingface_hub import HfApi
+        api = HfApi()
+        assert api.model_info(args.hub_model_id).private, "Private model required"
+        for name in ["results.json", "metrics.jsonl", "selected_row_ids.json.gz",
+                     "predict.py"]:
+            api.upload_file(path_or_fileobj=os.path.join(save_dir, name),
+                            repo_id=args.hub_model_id, repo_type="model",
+                            path_in_repo=name)
+    log_metric({"event": "persisted", "save_dir": save_dir, "pushed": args.push})
+    print("PROTOTYPE_DONE", flush=True)
+
+
+if __name__ == "__main__":
+    main()
+===== END FILE: recipe/train.py =====
+
+===== FILE: predict.py =====
+"""Typed-choice inference for ModernJEV-Decide-Preview.
+The encoder scores each declared (state, candidate) pair; it does not generate text.
+"""
+import json
+import os
+import torch
+from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+MAX_LEN = 4096
+MODEL_ID = "OpenMed/ModernJEV-Decide-Preview"
+
+def serialize_state(state):
+    if not isinstance(state, dict):
+        raise TypeError("state must be a dict containing conversation and available_tools")
+    conv = state.get("conversation") or []
+    policy = state.get("policy")
+    first = conv[0] if conv else None
+    duplicate = (policy is not None and isinstance(first, dict)
+                 and first.get("role") == "system" and first.get("content") == policy)
+    compact = {"available_tools": state.get("available_tools") or [], "conversation": conv}
+    if policy is not None and not duplicate:
+        compact["policy"] = policy
+    return json.dumps(compact, ensure_ascii=False)
+
+def normalize_criteria(criteria):
+    if isinstance(criteria, list):
+        if not criteria or any(not isinstance(v, str) or not v.strip() for v in criteria):
+            raise ValueError("Answer list must contain nonempty strings")
+        if len(set(criteria)) != len(criteria):
+            raise ValueError("Answer list must contain unique strings")
+        criteria = {v: v for v in criteria}
+    if not isinstance(criteria, dict) or not criteria:
+        raise ValueError("criteria must be a nonempty mapping or list of unique answer strings")
+    if any(not isinstance(k, str) or not k.strip() or not isinstance(v, str)
+           for k, v in criteria.items()):
+        raise TypeError("Choice labels must be nonempty strings and descriptions must be strings")
+    return criteria
+
+class DecisionModel:
+    def __init__(self, model_path=MODEL_ID, device=None, revision=None):
+        self.device = device or ("cuda" if torch.cuda.is_available() else "cpu")
+        self.tokenizer = AutoTokenizer.from_pretrained(model_path, revision=revision)
+        self.model = AutoModelForSequenceClassification.from_pretrained(
+            model_path, revision=revision, attn_implementation="sdpa").to(self.device).eval()
+        if self.model.config.num_labels != 1:
+            raise ValueError("Expected a trained scalar candidate-scoring head")
+
+    @torch.inference_mode()
+    def decide(self, *, state, question, criteria, candidate_batch_size=8):
+        if not isinstance(question, str) or not question.strip():
+            raise ValueError("question must be nonempty text")
+        criteria = normalize_criteria(criteria)
+        if not isinstance(candidate_batch_size, int) or isinstance(candidate_batch_size, bool) or candidate_batch_size < 1:
+            raise ValueError("candidate_batch_size must be a positive integer")
+        keys = list(criteria)
+        text_a = question + "\n\nSTATE:\n" + serialize_state(state)
+        text_bs = [f"{k}: {criteria[k]}" for k in keys]
+        raw_a_length = len(self.tokenizer(text_a, add_special_tokens=False, verbose=False)["input_ids"])
+        special = self.tokenizer.num_special_tokens_to_add(pair=True)
+        raw_lengths = [raw_a_length + len(self.tokenizer(t, add_special_tokens=False)["input_ids"]) + special for t in text_bs]
+        scores = []
+        for begin in range(0, len(keys), candidate_batch_size):
+            ts = text_bs[begin:begin + candidate_batch_size]
+            encoded = self.tokenizer([text_a] * len(ts), ts, truncation="only_first",
+                max_length=MAX_LEN, padding=True, return_tensors="pt", verbose=False).to(self.device)
+            with torch.autocast(device_type=self.device.split(":")[0],
+                                dtype=torch.bfloat16, enabled=self.device.startswith("cuda")):
+                logits = self.model(input_ids=encoded["input_ids"],
+                                    attention_mask=encoded["attention_mask"]).logits.squeeze(-1)
+            scores.extend(logits.float().cpu().tolist())
+        probabilities = torch.softmax(torch.tensor(scores), dim=0).tolist()
+        order = sorted(range(len(keys)), key=lambda i: (-scores[i], keys[i]))
+        return {"predicted_label": keys[order[0]], "allowed_choices": keys,
+            "candidates": [{"label": keys[i], "score": probabilities[i], "raw_score": scores[i],
+                            "rank": rank + 1} for rank,i in enumerate(order)],
+            "truncated": any(n > MAX_LEN for n in raw_lengths),
+            "max_sequence_length": MAX_LEN,
+            "note": "Scores rank this supplied choice set; they are not calibrated confidence."}
+
+_default = None
+def predict_typed(question_text, state_json, criteria_json):
+    global _default
+    if _default is None:
+        _default = DecisionModel(os.environ.get("MODEL_PATH", MODEL_ID))
+    state = json.loads(state_json) if isinstance(state_json, str) else state_json
+    criteria = json.loads(criteria_json) if isinstance(criteria_json, str) else criteria_json
+    return _default.decide(state=state, question=question_text, criteria=criteria)
+
+def predict_batch(rows):
+    return [predict_typed(r["question_text"], r["state_json"], r["criteria_json"]) for r in rows]
+
+
+===== END FILE: predict.py =====
+
+===== FILE: runtime-versions.json =====
+{
+  "torch": "2.12.0+cu126",
+  "transformers": "5.17.0",
+  "datasets": "5.0.1",
+  "accelerate": "1.15.0",
+  "huggingface-hub": "1.33.0",
+  "kernels": "0.16.0"
+}
+
+===== END FILE: runtime-versions.json =====
+
+===== FILE: evaluate_open_labels.py =====
+"""Supplemental frozen-checkpoint evaluation; no training or Hub writes.
+
+Run --audit-only first. After final checkpoint is saved, supply --model and,
+for a Hub model, --revision. CPU is default; JSONL predictions resume safely.
+"""
+import argparse
+import hashlib
+import importlib.util
+import json
+import statistics
+from collections import Counter
+from pathlib import Path
+
+ROOT = Path(__file__).resolve().parent
+DATA_REV = 'f2fb14e4ec977c420f376c08785664cd38763d7e'
+CACHE = Path('/private/tmp/atd-ml-intern-hub-cache/MaziyarPanahi___agent_tool_decisions-180_k/default/0.0.0') / DATA_REV
+FAMILIES = ('agent_next_action_type', 'tool_selection', 'when_to_call_tool')
+EXPECTED = {FAMILIES[0]: (1158, 616), FAMILIES[1]: (542, 79), FAMILIES[2]: (3652, 1295)}
+
+
+def write_json(path, value):
+    temp = path.with_suffix('.tmp')
+    temp.write_text(json.dumps(value, indent=2) + '\n')
+    temp.replace(path)
+
+
+def load_test():
+    from datasets import Dataset
+    return Dataset.from_file(str(CACHE / 'agent_tool_decisions-180_k-test.arrow'))
+
+
+def audit(test):
+    report = {'dataset_revision': DATA_REV, 'families': {}}
+    for family in FAMILIES:
+        rows = [r for r in test if r['task_family'] == family]
+        labels = Counter(r['gold_label'] for r in rows)
+        size, correct = EXPECTED[family]
+        assert len(rows) == size and max(labels.values()) == correct
+        candidates = [len(json.loads(r['criteria_json'])) for r in rows]
+        assert all(r['gold_label'] in json.loads(r['criteria_json']) for r in rows)
+        report['families'][family] = {
+            'n': size, 'majority_correct': correct, 'majority_accuracy': correct / size,
+            'majority_labels': sorted(k for k,v in labels.items() if v == correct),
+            'candidate_count': {'median': statistics.median(candidates),
+                                'min': min(candidates), 'max': max(candidates)},
+            'uniform_expected_accuracy': statistics.mean(1/n for n in candidates),
+            'reference_definition': 'Descriptive test-set constant-label majority; no model fitting',
+            'unseen_task_family': family == 'when_to_call_tool',
+        }
+    # Verify When2Call is absent from every training shard, not just the subset.
+    from datasets import Dataset
+    train_families = Counter()
+    selected_ids = set((ROOT / 'selected-row-ids.txt').read_text().splitlines())
+    train_gold = {f: Counter() for f in FAMILIES[:2]}
+    for path in sorted(CACHE.glob('*-train-*.arrow')):
+        columns = Dataset.from_file(str(path)).select_columns(['task_family','row_id','gold_label'])[:]
+        for family, row_id, gold in zip(columns['task_family'], columns['row_id'], columns['gold_label']):
+            train_families[family] += 1
+            if row_id in selected_ids and family in train_gold:
+                train_gold[family][gold] += 1
+    assert sum(train_families.values()) == 171056
+    assert train_families['when_to_call_tool'] == 0
+    assert sum(sum(c.values()) for c in train_gold.values()) == 60000
+    report['when2call_training_rows'] = 0
+    for family, counts in train_gold.items():
+        label = min(counts, key=lambda k: (-counts[k], k))
+        rows = [r for r in test if r['task_family'] == family]
+        report['families'][family]['training_fixed_majority'] = {
+            'label': label, 'correct': sum(r['gold_label'] == label for r in rows),
+            'n': len(rows),
+            'majority_label_absent_from_choices': sum(label not in json.loads(r['criteria_json']) for r in rows),
+        }
+    return report
+
+
+def open_labels(client):
+    checks = []
+    for n in (2, 3, 7, 20):
+        descriptions = ['Escalate this damaged parcel to a human support agent.',
+                        'Search the product catalog for a new item.']
+        descriptions += [f'Route to unrelated department number {i} for a different request.' for i in range(n-2)]
+        criteria = {f'fresh_choice_{n}_{i}': d for i,d in enumerate(descriptions)}
+        state = {'policy': 'Escalate damaged parcels to a human support agent.',
+                 'conversation': [{'role':'user','content':'My parcel arrived broken. I need help.'}]}
+        question = 'Which of these custom workflow branches should handle this request?'
+        outputs = []
+        variants = (criteria, dict(reversed(list(criteria.items()))),
+                    {f'opaque_{n}_{i}': d for i,d in enumerate(descriptions)}, descriptions)
+        for variant in variants:
+            answer = client.decide(state=state, question=question, criteria=variant, candidate_batch_size=4)
+            allowed = list(variant)
+            assert answer['predicted_label'] in allowed
+            assert sorted(c['label'] for c in answer['candidates']) == sorted(allowed)
+            assert len(answer['candidates']) == n
+            outputs.append(answer)
+        original_desc = criteria[outputs[0]['predicted_label']]
+        renamed_desc = variants[2][outputs[2]['predicted_label']]
+        checks.append({'answer_count': n, 'interface_pass': True,
+                       'reorder_same_label': outputs[0]['predicted_label'] == outputs[1]['predicted_label'],
+                       'rename_same_description': original_desc == renamed_desc,
+                       'selected_expected_description': original_desc == descriptions[0],
+                       'outputs': outputs})
+    return {'scalar_head': client.model.config.num_labels,
+            'interpretation': 'Interface checks and illustrative decisions, not a clinical benchmark or general accuracy estimate.',
+            'checks': checks}
+
+
+def main():
+    p = argparse.ArgumentParser()
+    p.add_argument('--audit-only', action='store_true')
+    p.add_argument('--interface-only', action='store_true')
+    p.add_argument('--model')
+    p.add_argument('--revision')
+    p.add_argument('--device', default='cpu')
+    p.add_argument('--output-dir', type=Path, default=ROOT / 'supplemental-evaluation')
+    args = p.parse_args()
+    out = args.output_dir; out.mkdir(parents=True, exist_ok=True)
+    test = load_test(); baseline = audit(test)
+    write_json(out/'per-task-baselines.json', baseline)
+    print(json.dumps(baseline), flush=True)
+    if args.audit_only: return
+    if not args.model: p.error('--model is required for checkpoint evaluation')
+    local = Path(args.model).is_dir()
+    if not local and not args.revision: p.error('Pin --revision for a Hub model')
+    if local:
+        weights = sorted(Path(args.model).glob('*.safetensors'))
+        assert weights, 'Local checkpoint must have saved safetensors'
+        h = hashlib.sha256()
+        for path in weights:
+            with path.open('rb') as f:
+                for chunk in iter(lambda:f.read(8*1024*1024), b''): h.update(chunk)
+        identity = h.hexdigest()
+    else: identity = args.revision
+    manifest = {'model': args.model, 'checkpoint_identity': identity, 'dataset_revision': DATA_REV}
+    if (out/'manifest.json').exists(): assert json.loads((out/'manifest.json').read_text()) == manifest
+    write_json(out/'manifest.json', manifest)
+    spec = importlib.util.spec_from_file_location('inference', ROOT/'predict_open_labels.py')
+    helper = importlib.util.module_from_spec(spec); spec.loader.exec_module(helper)
+    import torch
+    torch.set_num_threads(4)
+    client = helper.DecisionModel(args.model, device=args.device, revision=args.revision)
+    assert client.model.config.num_labels == 1
+    write_json(out/'open-label-checks.json', open_labels(client))
+    if args.interface_only: return
+    rows = [r for r in test if r['task_family'] == 'when_to_call_tool']
+    path = out/'when2call-predictions.jsonl'; previous = {}
+    if path.exists():
+        for line in path.read_text().splitlines():
+            if line: item = json.loads(line); previous[item['row_id']] = item
+    assert set(previous).issubset({r['row_id'] for r in rows})
+    with path.open('a') as f:
+        for row in rows:
+            if row['row_id'] in previous: continue
+            pred = client.decide(state=json.loads(row['state_json']), question=row['question_text'],
+                                 criteria=json.loads(row['criteria_json']), candidate_batch_size=4)
+            rec = {'row_id': row['row_id'], 'gold': row['gold_label'], **pred}
+            f.write(json.dumps(rec)+'\n'); f.flush(); previous[row['row_id']] = rec
+            if len(previous) % 100 == 0: print(f'When2Call {len(previous)}/{len(rows)}', flush=True)
+    correct = sum(previous[r['row_id']]['predicted_label'] == r['gold_label'] for r in rows)
+    n = len(rows); reference = baseline['families']['when_to_call_tool']['majority_accuracy']
+    result = {'task': 'when_to_call_tool', 'correct': correct, 'n': n, 'accuracy': correct/n,
+              'majority_reference': reference, 'lift_percentage_points': 100*(correct/n-reference),
+              'unseen_task_family': True, 'checkpoint_identity': identity,
+              'truncated_rows': sum(x['truncated'] for x in previous.values())}
+    write_json(out/'when2call-results.json', result)
+    print(json.dumps(result), flush=True)
+
+
+if __name__ == '__main__': main()
+
+===== END FILE: evaluate_open_labels.py =====

workflow/recipe-manifest.json

@@ -0,0 +1,19 @@
+[
+  {
+    "file": "recipe/train.py",
+    "sha256": "37774a1a9a7456cf25b2f33b7272a9817be89dc235577e4f63d507ad7128dd8a"
+  },
+  {
+    "file": "predict.py",
+    "sha256": "d77ea48b1fadc19b8e77404d6066031d571d01d7c2b6120526499b069bea2fd5"
+  },
+  {
+    "file": "runtime-versions.json",
+    "sha256": "a506eb8faa5a457b04834d03160d8054a16c3dba086e2d6d570d8e1dbc360163"
+  },
+  {
+    "file": "evaluate_open_labels.py",
+    "sha256": "da0db4b3114ec7d1d0fbfddb54668ef74f9ea940574d9c7f05f679af4ab8151e",
+    "source": "previously completed local evaluation runner"
+  }
+]