Jev Support Ticket Routing Benchmark

import { experimental_evaluate } from '@ai-sdk/typesafe';

const TICKET_ROUTING_SCHEMA = {
  department: {
    type: 'choice',
    instructions: 'Determine the primary department responsible for resolving this customer ticket.',
    criteria: {
      billing: 'Questions regarding invoices, payment failures, duplicate charges, or refunds.',
      technical: 'Software bugs, API failures, service outages, or integration errors.',
      sales: 'Inquiries about plan upgrades, enterprise pricing, seat additions, or contract renewals.',
      abuse: 'Reports of phishing, spam, Terms of Service violations, or malicious activity.',
    },
  },
};

export async function runRoutingEval(ticket: string) {
  const t0 = performance.now();
  const result = await experimental_evaluate({
    model: 'jev-latest',
    state: { ticketText: ticket },
    questions: TICKET_ROUTING_SCHEMA,
  });
  const durationMs = Math.round(performance.now() - t0);

  return {
    choice: result.answers.department.value,
    confidence: result.providerMetadata?.typesafe?.confidence?.department ?? result.answers.department.confidence,
    probabilities: result.answers.department.probabilities,
    durationMs,
  };
}