用 Jev 做决策弃答：教会 AI 说「我不知道」

import os

import requests

resp = requests.post(
    "https://api.openai.com/v1/chat/completions",
    headers={"Authorization": f"Bearer {os.environ['OPENAI_API_KEY']}"},
    json={
        "model": "gpt-4o-mini",
        "messages": [{
            "role": "user",
            "content": (
                "订单 #9012 什么时候发货？只使用下面的数据。"
                "如果数据里没有答案，一字不差地回复：I DON'T KNOW\n"
                "数据：{'lines': [...], 'channel': 'web'}"
            ),
        }],
    },
    timeout=30,
)
text = resp.json()["choices"][0]["message"]["content"]

if "I DON'T KNOW" in text:
    ...

# 你看不见的失败：把问法换成「最晚哪天发货」、语气放软一点、
# 或者数据贴长一点，同一个 state 就会回你「10 月 14 日发货」——
# 流利、无法证伪、和有根据的答案无法区分。你门控的是散文，
# 而散文没有阈值。

import requests

JEV_ENDPOINT = "https://api.typesafe.ai/v1/jev/evaluate"
ANSWERABILITY_GATE = 0.60  # is_answerable 低于此 -> 弃答
AUTO_CONFIDENCE = 0.85     # 答案置信度低于此 -> 人工复核

QUESTIONS = {
    "answer": {
        "type": "choice",
        "instructions": "只使用 state 里已有的信息作答",
        "criteria": {
            "billing": "付款、发票或退款",
            "technical": "产品故障",
            "sales": "购买或升级咨询",
        },
    },
    "is_answerable": {
        "type": "noul",
        "instructions": "state 是否包含选出唯一选项所需的全部信息——不是多给点数据，不是猜？",
    },
    "missing_information": {
        "type": "choice",
        "instructions": "若 is_answerable 为否：缺失的究竟是哪一样？",
        "criteria": {
            "none": "state 信息充分",
            "a_date": "一个日期或截止时间",
            "an_amount": "一个金额或余额",
            "an_identity": "某个人或账户是谁",
            "a_policy_rule": "适用哪条政策或哪个版本",
            "external_state": "另一个系统里的实时数据",
        },
    },
}

def decide_or_abstain(state: dict) -> dict:
    resp = requests.post(
        JEV_ENDPOINT,
        json={"state": state, "questions": QUESTIONS},
        timeout=5,
    )
    resp.raise_for_status()
    data = resp.json()

    knows = data["is_answerable"]
    if not knows["answer"] or knows["confidence"] < ANSWERABILITY_GATE:
        # 类型化弃答：工作流按缺失键路由，
        # 下面这个对象可记录、可计数、可重放。
        return {
            "abstained": True,
            "missing": data["missing_information"]["answer"],
            "is_answerable": knows["confidence"],
        }

    answer = data["answer"]
    lane = "auto" if answer["confidence"] >= AUTO_CONFIDENCE else "review"
    return {
        "abstained": False,
        "answer": answer["answer"],
        "confidence": answer["confidence"],
        "lane": lane,
    }

import OpenAI from "openai";

const openai = new OpenAI();

export async function whenDoesItShip(orderData: object) {
  const resp = await openai.chat.completions.create({
    model: "gpt-4o-mini",
    messages: [
      {
        role: "user",
        content: `这张订单什么时候发货？只使用下面的数据。如果数据里没有答案，一字不差地回复：I DON'T KNOW\n数据：${JSON.stringify(orderData)}`,
      },
    ],
  });
  const text = resp.choices[0].message.content ?? "";

  // 你门控的是散文。换个问法、语气软一点、数据贴长一点，
  // 拒绝行为就会漂移——而且这个返回类型里没有任何东西
  // 能区分有根据的日期和一个自信的瞎猜。
  return { text, refused: text.includes("I DON'T KNOW") };
}

const JEV_ENDPOINT = "https://api.typesafe.ai/v1/jev/evaluate";
const ANSWERABILITY_GATE = 0.6; // is_answerable 低于此 -> 弃答
const AUTO_CONFIDENCE = 0.85; // 答案置信度低于此 -> 复核

const QUESTIONS = {
  answer: {
    type: "choice",
    instructions: "只使用 state 里已有的信息作答",
    criteria: {
      billing: "付款、发票或退款",
      technical: "产品故障",
      sales: "购买或升级咨询",
    },
  },
  is_answerable: {
    type: "noul",
    instructions: "state 是否包含选出唯一选项所需的全部信息——不是多给点数据，不是猜？",
  },
  missing_information: {
    type: "choice",
    instructions: "若 is_answerable 为否：缺失的究竟是哪一样？",
    criteria: {
      none: "state 信息充分",
      a_date: "一个日期或截止时间",
      an_amount: "一个金额或余额",
      an_identity: "某个人或账户是谁",
      a_policy_rule: "适用哪条政策或哪个版本",
      external_state: "另一个系统里的实时数据",
    },
  },
} as const;

export type Outcome =
  | { abstained: true; missing: string; is_answerable: number }
  | {
      abstained: false;
      answer: string;
      confidence: number;
      lane: "auto" | "review";
    };

export async function decideOrAbstain(state: object): Promise<Outcome> {
  const resp = await fetch(JEV_ENDPOINT, {
    method: "POST",
    headers: { "Content-Type": "application/json" },
    body: JSON.stringify({ state, questions: QUESTIONS }),
  });
  if (!resp.ok) throw new Error("Jev evaluate failed");
  const data = await resp.json();

  const knows = data.is_answerable as { answer: boolean; confidence: number };
  if (!knows.answer || knows.confidence < ANSWERABILITY_GATE) {
    return {
      abstained: true,
      missing: (data.missing_information as { answer: string }).answer,
      is_answerable: knows.confidence,
    };
  }

  const answer = data.answer as { answer: string; confidence: number };
  return {
    abstained: false,
    answer: answer.answer,
    confidence: answer.confidence,
    lane: answer.confidence >= AUTO_CONFIDENCE ? "auto" : "review",
  };
}
// 一次调用，三个类型化读数。弃答分支带着缺失键，
// 工作流可以精确去补那一样东西。