Decision Model Abstention with Jev: When the Right Answer Is "I Don't Know"

import os

import requests

resp = requests.post(
    "https://api.openai.com/v1/chat/completions",
    headers={"Authorization": f"Bearer {os.environ['OPENAI_API_KEY']}"},
    json={
        "model": "gpt-4o-mini",
        "messages": [{
            "role": "user",
            "content": (
                "When will order #9012 ship? Use only the data below. "
                "If the data does not contain the answer, reply exactly: I DON'T KNOW\n"
                "Data: {'lines': [...], 'channel': 'web'}"
            ),
        }],
    },
    timeout=30,
)
text = resp.json()["choices"][0]["message"]["content"]

if "I DON'T KNOW" in text:
    ...

# The failure you cannot see: rephrase the question ("by which date..."),
# warm up the style, or lengthen the data dump, and the same state returns
# "Ships Oct 14" — fluent, unfalsifiable, and indistinguishable from a
# grounded answer. You gated on prose, and prose has no threshold.

import requests

JEV_ENDPOINT = "https://api.typesafe.ai/v1/jev/evaluate"
ANSWERABILITY_GATE = 0.60  # is_answerable below this -> abstain
AUTO_CONFIDENCE = 0.85     # answer confidence below this -> review

QUESTIONS = {
    "answer": {
        "type": "choice",
        "instructions": "Answer using only what the state contains",
        "criteria": {
            "billing": "Payment, invoice or refund",
            "technical": "Product malfunction",
            "sales": "Buying or upgrade question",
        },
    },
    "is_answerable": {
        "type": "noul",
        "instructions": "Does the state contain enough information to pick exactly one option — not more data, not a guess?",
    },
    "missing_information": {
        "type": "choice",
        "instructions": "If is_answerable is no: what single piece is missing?",
        "criteria": {
            "none": "The state is sufficient",
            "a_date": "A date or deadline",
            "an_amount": "An amount or balance",
            "an_identity": "Who a person or account is",
            "a_policy_rule": "Which policy or version applies",
            "external_state": "Live data from another system",
        },
    },
}

def decide_or_abstain(state: dict) -> dict:
    resp = requests.post(
        JEV_ENDPOINT,
        json={"state": state, "questions": QUESTIONS},
        timeout=5,
    )
    resp.raise_for_status()
    data = resp.json()

    knows = data["is_answerable"]
    if not knows["answer"] or knows["confidence"] < ANSWERABILITY_GATE:
        # Typed abstention: the workflow routes on the missing key,
        # and the record below is loggable, countable, replayable.
        return {
            "abstained": True,
            "missing": data["missing_information"]["answer"],
            "is_answerable": knows["confidence"],
        }

    answer = data["answer"]
    lane = "auto" if answer["confidence"] >= AUTO_CONFIDENCE else "review"
    return {
        "abstained": False,
        "answer": answer["answer"],
        "confidence": answer["confidence"],
        "lane": lane,
    }

import OpenAI from "openai";

const openai = new OpenAI();

export async function whenDoesItShip(orderData: object) {
  const resp = await openai.chat.completions.create({
    model: "gpt-4o-mini",
    messages: [
      {
        role: "user",
        content: `When will this order ship? Use only the data below. If the data does not contain the answer, reply exactly: I DON'T KNOW\nData: ${JSON.stringify(orderData)}`,
      },
    ],
  });
  const text = resp.choices[0].message.content ?? "";

  // You gated on prose. A paraphrased question, a warmer style, or a
  // longer data dump shifts refusal behavior — and nothing in this
  // return type can tell a grounded date from a confident guess.
  return { text, refused: text.includes("I DON'T KNOW") };
}

const JEV_ENDPOINT = "https://api.typesafe.ai/v1/jev/evaluate";
const ANSWERABILITY_GATE = 0.6; // is_answerable below -> abstain
const AUTO_CONFIDENCE = 0.85; // answer confidence below -> review

const QUESTIONS = {
  answer: {
    type: "choice",
    instructions: "Answer using only what the state contains",
    criteria: {
      billing: "Payment, invoice or refund",
      technical: "Product malfunction",
      sales: "Buying or upgrade question",
    },
  },
  is_answerable: {
    type: "noul",
    instructions: "Does the state contain enough information to pick exactly one option — not more data, not a guess?",
  },
  missing_information: {
    type: "choice",
    instructions: "If is_answerable is no: what single piece is missing?",
    criteria: {
      none: "The state is sufficient",
      a_date: "A date or deadline",
      an_amount: "An amount or balance",
      an_identity: "Who a person or account is",
      a_policy_rule: "Which policy or version applies",
      external_state: "Live data from another system",
    },
  },
} as const;

export type Outcome =
  | { abstained: true; missing: string; is_answerable: number }
  | {
      abstained: false;
      answer: string;
      confidence: number;
      lane: "auto" | "review";
    };

export async function decideOrAbstain(state: object): Promise<Outcome> {
  const resp = await fetch(JEV_ENDPOINT, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({ state, questions: QUESTIONS }),
  });
  if (!resp.ok) throw new Error("Jev evaluate failed");
  const data = await resp.json();

  const knows = data.is_answerable as { answer: boolean; confidence: number };
  if (!knows.answer || knows.confidence < ANSWERABILITY_GATE) {
    return {
      abstained: true,
      missing: (data.missing_information as { answer: string }).answer,
      is_answerable: knows.confidence,
    };
  }

  const answer = data.answer as { answer: string; confidence: number };
  return {
    abstained: false,
    answer: answer.answer,
    confidence: answer.confidence,
    lane: answer.confidence >= AUTO_CONFIDENCE ? "auto" : "review",
  };
}
// One call, three typed readings. The abstention branch carries the
// missing key, so the workflow can go collect exactly that piece.