The evaluation loop

Question types

The score

JevEval = Σ wᵢ·vᵢ / Σ wᵢ     over applicable questions only

TypeScript: the arithmetic

From lesson 1, where every line was run against the docs' worked example and printed their 0.461.

src/jev-eval.tsthe scoring functions

type Question = { name: string; weight: number; value: number | null };

// A weighted mean of every question's value.
function jevEval(questions: Question[]): number {
  let weighted = 0;
  let totalWeight = 0;
  for (const q of questions) {
    if (q.value === null) continue; // this question doesn't apply
    weighted += q.weight * q.value;
    totalWeight += q.weight;
  }
  if (totalWeight === 0) return 1; // nothing applicable was left to fail
  return weighted / totalWeight;
}

// A Score's value is the expected level, counted from 0, over the top level.
function scoreValue(levels: string[], answer: Record<string, number>): number {
  let expected = 0;
  for (const [level, label] of levels.entries()) {
    expected += level * (answer[label] ?? 0);
  }
  return expected / (levels.length - 1);
}

type Credits = Record<string, number | null>;

// A Choice's value is the credit Jev's answer earns, averaged over the options
// that count. When the null options hold half the probability or more, the
// question doesn't apply, and the value is null.
function choiceValue(
  credits: Credits,
  answer: Record<string, number>
): number | null {
  let noneMass = 0;
  let mass = 0;
  let earned = 0;
  for (const [option, credit] of Object.entries(credits)) {
    const p = answer[option] ?? 0;
    if (credit === null) {
      noneMass += p;
      continue;
    }
    mass += p;
    earned += p * credit;
  }
  if (noneMass >= 0.5) return null;
  return earned / mass;
}

const missingCredits: Credits = {
  left_it_out: 1,
  flagged_it_as_unknown: 1,
  hedged_it: 0.5,
  stated_it_as_fact: 0,
  nothing_missing: null,
};

Python: deepeval

pip install typesafe-sdk          # the docs' command; deepeval itself as well
export TYPESAFE_API_KEY=...

from deepeval.metrics.jev_eval import Noul, Score, Choice
from deepeval.metrics import JevEval, ConversationalJevEval
from deepeval.test_case import SingleTurnParams, MultiTurnParams

metric.measure(test_case)
print(metric.score, metric.reason)
print(metric.score_breakdown)
# each entry: question, type, weight, value, applicable, probabilities, confidence

from deepeval import evaluate
evaluate(test_cases=[test_case], metrics=[metric])

Questions about a conversation talk about turns as a whole ("by the end of turns", "at any point in turns") or about a role ("the assistant", "the user").

The worked example