{
  "title": "KcalSnap Jev food-matching experiment",
  "date": "2026-09-18",
  "article": "https://kcalsnap.com/journal/jev-food-matcher/",
  "baseline": "Production matcher before the four September 18 fixes; these are not scores for the updated code.",
  "limits": [
    "Internal development experiment, not an independent evaluation.",
    "Matcher labels are semantic judgments on a trap-heavy set. Repeated decisions are correlated.",
    "Nine public photos with visually estimated reference portions; reference calories are estimates, not measured truth.",
    "Local timing excludes phone upload, server startup, database food-cache hits and rendering. Most USDA retrieval used a warm raw cache.",
    "Each photo arm uses fresh AI macro estimates, which add variance."
  ],
  "models": {
    "jevRequested": "typesafe/jev-1.13",
    "jevResolved": "typesafe/jev-1.13-20260917",
    "comparison": "google/gemini-3.5-flash-lite",
    "photoVision": "google/gemini-3.1-flash-lite"
  },
  "matcher": {
    "uniqueQueries": 60,
    "developmentQueries": 39,
    "heldOutQueries": 21,
    "modelPasses": 3,
    "metrics": [
      {
        "split": "all",
        "arm": "code",
        "decisions": 60,
        "uniqueCases": 60,
        "correct": 44,
        "wrongFood": 7,
        "falseReject": 9,
        "correctNone": 13,
        "trueNoneDecisions": 14,
        "vetoed": 0,
        "timedOut": 0,
        "p50Ms": 0,
        "p95Ms": 0
      },
      {
        "split": "all",
        "arm": "jev-full",
        "decisions": 180,
        "uniqueCases": 60,
        "correct": 168,
        "wrongFood": 9,
        "falseReject": 3,
        "correctNone": 33,
        "trueNoneDecisions": 42,
        "vetoed": 0,
        "timedOut": 0,
        "p50Ms": 333,
        "p95Ms": 732
      },
      {
        "split": "all",
        "arm": "flash35-full",
        "decisions": 180,
        "uniqueCases": 60,
        "correct": 164,
        "wrongFood": 13,
        "falseReject": 3,
        "correctNone": 36,
        "trueNoneDecisions": 42,
        "vetoed": 0,
        "timedOut": 0,
        "p50Ms": 585,
        "p95Ms": 774
      },
      {
        "split": "holdout",
        "arm": "code",
        "decisions": 21,
        "uniqueCases": 21,
        "correct": 12,
        "wrongFood": 4,
        "falseReject": 5,
        "correctNone": 4,
        "trueNoneDecisions": 5,
        "vetoed": 0,
        "timedOut": 0,
        "p50Ms": 0,
        "p95Ms": 0
      },
      {
        "split": "holdout",
        "arm": "jev-full",
        "decisions": 63,
        "uniqueCases": 21,
        "correct": 60,
        "wrongFood": 3,
        "falseReject": 0,
        "correctNone": 12,
        "trueNoneDecisions": 15,
        "vetoed": 0,
        "timedOut": 0,
        "p50Ms": 337,
        "p95Ms": 779
      },
      {
        "split": "holdout",
        "arm": "flash35-full",
        "decisions": 63,
        "uniqueCases": 21,
        "correct": 57,
        "wrongFood": 6,
        "falseReject": 0,
        "correctNone": 12,
        "trueNoneDecisions": 15,
        "vetoed": 0,
        "timedOut": 0,
        "p50Ms": 591,
        "p95Ms": 693
      }
    ]
  },
  "photoTest": {
    "uniquePhotos": 9,
    "passes": 3,
    "itemsPerArm": 151,
    "selectorDeadlineMs": 1500,
    "onTimeout": "Use the exact baseline code decision.",
    "metrics": [
      {
        "arm": "code",
        "scans": 27,
        "photos": 9,
        "totalP50Ms": 2690,
        "totalP95Ms": 5599,
        "selectorCalls": 0,
        "timeouts": 0,
        "aiEstimatedItems": 38,
        "items": 151,
        "costPerScanUsd": 0.000910203703703704,
        "unresolvedReservePerScanUsd": 0,
        "within25PctOfEstimatedReference": 21,
        "medianAbsoluteReferenceDeviationPct": 10.888875640872786,
        "meanAbsoluteReferenceDeviationPct": 15.937582452463666
      },
      {
        "arm": "jev-full",
        "scans": 27,
        "photos": 9,
        "totalP50Ms": 2749,
        "totalP95Ms": 5969,
        "selectorCalls": 151,
        "timeouts": 2,
        "aiEstimatedItems": 7,
        "items": 151,
        "costPerScanUsd": 0.0013511915555555558,
        "unresolvedReservePerScanUsd": 1.907266666666667e-05,
        "within25PctOfEstimatedReference": 22,
        "medianAbsoluteReferenceDeviationPct": 15.220483641536257,
        "meanAbsoluteReferenceDeviationPct": 15.822904911316552
      },
      {
        "arm": "flash35-full",
        "scans": 27,
        "photos": 9,
        "totalP50Ms": 2666,
        "totalP95Ms": 6387,
        "selectorCalls": 151,
        "timeouts": 0,
        "aiEstimatedItems": 14,
        "items": 151,
        "costPerScanUsd": 0.00330124074074074,
        "unresolvedReservePerScanUsd": 0,
        "within25PctOfEstimatedReference": 23,
        "medianAbsoluteReferenceDeviationPct": 18.419753086419757,
        "meanAbsoluteReferenceDeviationPct": 16.233813809856436
      },
      {
        "arm": "jev-veto",
        "scans": 27,
        "photos": 9,
        "totalP50Ms": 3098,
        "totalP95Ms": 6468,
        "selectorCalls": 113,
        "timeouts": 1,
        "aiEstimatedItems": 41,
        "items": 151,
        "costPerScanUsd": 0.0013481961481481486,
        "unresolvedReservePerScanUsd": 6.650000000000001e-06,
        "within25PctOfEstimatedReference": 22,
        "medianAbsoluteReferenceDeviationPct": 14.537695590327152,
        "meanAbsoluteReferenceDeviationPct": 15.315897946297039
      },
      {
        "arm": "flash35-veto",
        "scans": 27,
        "photos": 9,
        "totalP50Ms": 3376,
        "totalP95Ms": 6714,
        "selectorCalls": 113,
        "timeouts": 0,
        "aiEstimatedItems": 42,
        "items": 151,
        "costPerScanUsd": 0.0027974314814814823,
        "unresolvedReservePerScanUsd": 0,
        "within25PctOfEstimatedReference": 22,
        "medianAbsoluteReferenceDeviationPct": 16.345679012345677,
        "meanAbsoluteReferenceDeviationPct": 15.985010203791958
      }
    ],
    "pairedComparisons": [
      {
        "a": "jev-full",
        "b": "code",
        "n": 27,
        "faster": 10,
        "medianTotalDeltaMs": 330,
        "medianTotalDeltaPhotoCluster95": [
          -501,
          602
        ],
        "medianSelectionDeltaMs": 406
      },
      {
        "a": "flash35-full",
        "b": "code",
        "n": 27,
        "faster": 7,
        "medianTotalDeltaMs": 579,
        "medianTotalDeltaPhotoCluster95": [
          -210,
          640
        ],
        "medianSelectionDeltaMs": 644
      },
      {
        "a": "jev-veto",
        "b": "code",
        "n": 27,
        "faster": 1,
        "medianTotalDeltaMs": 450,
        "medianTotalDeltaPhotoCluster95": [
          374,
          952
        ],
        "medianSelectionDeltaMs": 389
      },
      {
        "a": "flash35-veto",
        "b": "code",
        "n": 27,
        "faster": 0,
        "medianTotalDeltaMs": 630,
        "medianTotalDeltaPhotoCluster95": [
          569,
          684
        ],
        "medianSelectionDeltaMs": 639
      },
      {
        "a": "jev-full",
        "b": "flash35-full",
        "n": 27,
        "faster": 21,
        "medianTotalDeltaMs": -275,
        "medianTotalDeltaPhotoCluster95": [
          -337,
          -232
        ],
        "medianSelectionDeltaMs": -233
      },
      {
        "a": "jev-veto",
        "b": "flash35-veto",
        "n": 27,
        "faster": 15,
        "medianTotalDeltaMs": -234,
        "medianTotalDeltaPhotoCluster95": [
          -271,
          93
        ],
        "medianSelectionDeltaMs": -241
      }
    ]
  },
  "researchAssistance": {
    "orchestration": {
      "model": "Astra",
      "effort": "xhigh"
    },
    "reviewRecommendations": {
      "model": "Fable 5.1",
      "effort": "medium"
    },
    "scope": "Research assistance, not additional food-selector benchmark arms. KcalSnap is responsible for interpretation and the production decision."
  }
}