{ "complete": true, "photos": 48, "repeats": 2, "matcher": "v4", "scope": "Production prompts, parsing, matcher and same-model macro fallback. Warm local USDA pools; not deployed mobile UX.", "referenceLimits": "Measured portions and USDA-derived nutrient references. Cafeteria/camera bias and possible model-training exposure. Inferred grouping and reference consistency filtering.", "summaries": [ { "model": "google/gemini-3.1-flash-lite", "expected": 96, "completed": 96, "valid": 96, "failures": 0, "within25": 48, "within25PctAllCompleted": 50, "metrics": { "kcal": { "mae": 102.96032575000002, "wapePct": 27.997584757986008, "medianApePct": 25.12154471757527, "meanApePct": 35.29639037327546, "meanSignedError": 17.9085586875, "nonzeroReferenceCount": 96 }, "protein_g": { "mae": 7.634513749999997, "wapePct": 29.713568039626914, "medianApePct": 19.726210183565854, "meanApePct": 39.93725945944667, "meanSignedError": -2.354610270833333, "nonzeroReferenceCount": 96 }, "carbs_g": { "mae": 11.21314425, "wapePct": 40.70842451120864, "medianApePct": 42.17584075773365, "meanApePct": 66.8598354924982, "meanSignedError": 3.744428583333334, "nonzeroReferenceCount": 96 }, "fat_g": { "mae": 8.614712520833335, "wapePct": 47.9804467770789, "medianApePct": 39.170980479593794, "meanApePct": 86.85452463568669, "meanSignedError": 0.9584443124999997, "nonzeroReferenceCount": 96 }, "grams": { "mae": 75.55208333333333, "wapePct": 27.06747275712793, "medianApePct": 22.884525479195887, "meanApePct": 27.979489227832925, "meanSignedError": -16.71875, "nonzeroReferenceCount": 96 } }, "visionP50Ms": 1777, "visionP95Ms": 2326.5, "warmPipelineP50Ms": 2037, "warmPipelineP95Ms": 3364.25, "totalCostUsd": 0.06712149999999999, "costPerCompletedScanUsd": 0.0006991822916666665, "itemCount": 384, "usdaItems": 341, "aiItems": 43 }, { "model": "google/gemini-3.5-flash-lite", "expected": 96, "completed": 96, "valid": 96, "failures": 0, "within25": 52, "within25PctAllCompleted": 54.166666666666664, "metrics": { "kcal": { "mae": 94.88077735416664, "wapePct": 25.800545856148666, "medianApePct": 20.25321393827903, "meanApePct": 26.572421032597347, "meanSignedError": -15.600972562500004, "nonzeroReferenceCount": 96 }, "protein_g": { "mae": 8.534760604166665, "wapePct": 33.217333574627936, "medianApePct": 25.210446057365676, "meanApePct": 44.63994202242208, "meanSignedError": -3.8804993333333324, "nonzeroReferenceCount": 96 }, "carbs_g": { "mae": 9.209151875000002, "wapePct": 33.43309026954622, "medianApePct": 28.886796498961914, "meanApePct": 43.56614754366016, "meanSignedError": 1.2800640000000003, "nonzeroReferenceCount": 96 }, "fat_g": { "mae": 7.400403416666664, "wapePct": 41.2172386952664, "medianApePct": 33.702641640914095, "meanApePct": 63.80314680515545, "meanSignedError": -0.9578931875000002, "nonzeroReferenceCount": 96 }, "grams": { "mae": 67.29166666666667, "wapePct": 24.108075832213764, "medianApePct": 19.201228878648234, "meanApePct": 23.37305191314674, "meanSignedError": -27.583333333333332, "nonzeroReferenceCount": 96 } }, "visionP50Ms": 1763, "visionP95Ms": 2492, "warmPipelineP50Ms": 1941.5, "warmPipelineP95Ms": 3301.5, "totalCostUsd": 0.09451099999999998, "costPerCompletedScanUsd": 0.000984489583333333, "itemCount": 355, "usdaItems": 312, "aiItems": 43 }, { "model": "google/gemini-3.8-flash", "expected": 96, "completed": 96, "valid": 95, "failures": 1, "within25": 47, "within25PctAllCompleted": 48.95833333333333, "metrics": { "kcal": { "mae": 95.7621312526316, "wapePct": 26.145207191792274, "medianApePct": 25.551490232111217, "meanApePct": 27.091097335571707, "meanSignedError": -46.83904615789473, "nonzeroReferenceCount": 95 }, "protein_g": { "mae": 7.995193157894737, "wapePct": 30.877794588766676, "medianApePct": 22.7100523643288, "meanApePct": 34.28818993387457, "meanSignedError": -4.507652484210525, "nonzeroReferenceCount": 95 }, "carbs_g": { "mae": 8.823169894736848, "wapePct": 32.36247784705115, "medianApePct": 30.85055123130311, "meanApePct": 40.68210098802514, "meanSignedError": -0.4899795578947367, "nonzeroReferenceCount": 95 }, "fat_g": { "mae": 7.212276357894734, "wapePct": 40.488413760780226, "medianApePct": 39.697415100658965, "meanApePct": 51.501607705960815, "meanSignedError": -3.414968926315789, "nonzeroReferenceCount": 95 }, "grams": { "mae": 68.49473684210527, "wapePct": 24.382658223104883, "medianApePct": 19.35483870967742, "meanApePct": 23.631737507710792, "meanSignedError": -31.08421052631579, "nonzeroReferenceCount": 95 } }, "visionP50Ms": 4347.5, "visionP95Ms": 8077, "warmPipelineP50Ms": 4893, "warmPipelineP95Ms": 15861.8, "totalCostUsd": 0.4408005, "costPerCompletedScanUsd": 0.004591671875, "itemCount": 378, "usdaItems": 340, "aiItems": 38 } ], "comparisons": [ { "model": "google/gemini-3.5-flash-lite", "baseline": "google/gemini-3.1-flash-lite", "pairedPlates": 48, "kcalMeanApeDeltaPp": -8.723969340678105, "plateBootstrap95Pp": [ -19.77632525357436, 1.0261404834436387 ], "medianPairedPipelineDeltaMs": -28.75, "qualifies": false, "plateDiffs": [ { "dishId": "dish_1566414291", "kcalApeDeltaPp": -7.96773224896457, "pipelineDeltaMs": -1279 }, { "dishId": "dish_1562776497", "kcalApeDeltaPp": 1.8161105301042255, "pipelineDeltaMs": -32.5 }, { "dishId": "dish_1560801883", "kcalApeDeltaPp": 8.352676997438547, "pipelineDeltaMs": -1443.5 }, { "dishId": "dish_1563221157", "kcalApeDeltaPp": 35.71117802691504, "pipelineDeltaMs": 1106 }, { "dishId": "dish_1557862783", "kcalApeDeltaPp": -16.005237979377767, "pipelineDeltaMs": 873 }, { "dishId": "dish_1565894910", "kcalApeDeltaPp": -20.442624258997455, "pipelineDeltaMs": 731.5 }, { "dishId": "dish_1564773976", "kcalApeDeltaPp": 16.73236128251389, "pipelineDeltaMs": -25 }, { "dishId": "dish_1561148042", "kcalApeDeltaPp": -26.742694280947134, "pipelineDeltaMs": -77 }, { "dishId": "dish_1564686966", "kcalApeDeltaPp": -55.34465173740723, "pipelineDeltaMs": -111.5 }, { "dishId": "dish_1562617703", "kcalApeDeltaPp": -2.427216989722856, "pipelineDeltaMs": 153.5 }, { "dishId": "dish_1565711570", "kcalApeDeltaPp": -4.20198461026448, "pipelineDeltaMs": -3 }, { "dishId": "dish_1558373159", "kcalApeDeltaPp": -1.7279542124620484, "pipelineDeltaMs": 439.5 }, { "dishId": "dish_1561577848", "kcalApeDeltaPp": -3.9917950950128205, "pipelineDeltaMs": -187 }, { "dishId": "dish_1560368570", "kcalApeDeltaPp": -4.712800337524278, "pipelineDeltaMs": 359.5 }, { "dishId": "dish_1566587143", "kcalApeDeltaPp": 5.770011857665297, "pipelineDeltaMs": -753 }, { "dishId": "dish_1561405411", "kcalApeDeltaPp": -0.9746699725125172, "pipelineDeltaMs": 113 }, { "dishId": "dish_1566316757", "kcalApeDeltaPp": 30.792238956031134, "pipelineDeltaMs": 191.5 }, { "dishId": "dish_1564588919", "kcalApeDeltaPp": -1.250207687031164, "pipelineDeltaMs": -119 }, { "dishId": "dish_1562876531", "kcalApeDeltaPp": -125.63018484665359, "pipelineDeltaMs": -223.5 }, { "dishId": "dish_1558114284", "kcalApeDeltaPp": -12.853663678787527, "pipelineDeltaMs": 61.5 }, { "dishId": "dish_1565638599", "kcalApeDeltaPp": 28.11241115244004, "pipelineDeltaMs": -496.5 }, { "dishId": "dish_1561662216", "kcalApeDeltaPp": 6.84853446399136, "pipelineDeltaMs": -120.5 }, { "dishId": "dish_1565030350", "kcalApeDeltaPp": -3.7548535676613284, "pipelineDeltaMs": -10.5 }, { "dishId": "dish_1558113941", "kcalApeDeltaPp": 8.024857285954877, "pipelineDeltaMs": 8 }, { "dishId": "dish_1561663606", "kcalApeDeltaPp": -18.428367087714804, "pipelineDeltaMs": -162 }, { "dishId": "dish_1559850338", "kcalApeDeltaPp": 24.781762979167386, "pipelineDeltaMs": 57 }, { "dishId": "dish_1565898128", "kcalApeDeltaPp": 38.27187885388851, "pipelineDeltaMs": -947 }, { "dishId": "dish_1565118782", "kcalApeDeltaPp": -4.125400657566615, "pipelineDeltaMs": 1922 }, { "dishId": "dish_1565898158", "kcalApeDeltaPp": -7.933529965348855, "pipelineDeltaMs": 186.5 }, { "dishId": "dish_1560800951", "kcalApeDeltaPp": -157.0121052972729, "pipelineDeltaMs": -197 }, { "dishId": "dish_1561739265", "kcalApeDeltaPp": -19.12409200069569, "pipelineDeltaMs": -346 }, { "dishId": "dish_1562774206", "kcalApeDeltaPp": 3.213243382601389, "pipelineDeltaMs": 202 }, { "dishId": "dish_1563295044", "kcalApeDeltaPp": -56.255571331449374, "pipelineDeltaMs": 236 }, { "dishId": "dish_1559239369", "kcalApeDeltaPp": 6.912618498512354, "pipelineDeltaMs": -466 }, { "dishId": "dish_1566502573", "kcalApeDeltaPp": -11.783176986094922, "pipelineDeltaMs": -262 }, { "dishId": "dish_1565123881", "kcalApeDeltaPp": 18.368795068371792, "pipelineDeltaMs": 137 }, { "dishId": "dish_1558461792", "kcalApeDeltaPp": -2.3368580060423, "pipelineDeltaMs": 105.5 }, { "dishId": "dish_1560203424", "kcalApeDeltaPp": 0.9518454776353629, "pipelineDeltaMs": -314 }, { "dishId": "dish_1562099157", "kcalApeDeltaPp": -77.12643277212064, "pipelineDeltaMs": -397.5 }, { "dishId": "dish_1565898230", "kcalApeDeltaPp": -11.285538497178933, "pipelineDeltaMs": -347 }, { "dishId": "dish_1561662814", "kcalApeDeltaPp": -59.63596546459489, "pipelineDeltaMs": -86 }, { "dishId": "dish_1566501594", "kcalApeDeltaPp": 19.658537608601158, "pipelineDeltaMs": -248.5 }, { "dishId": "dish_1563984296", "kcalApeDeltaPp": 9.424854570056489, "pipelineDeltaMs": 150.5 }, { "dishId": "dish_1567785778", "kcalApeDeltaPp": 0.41185142745085557, "pipelineDeltaMs": 816.5 }, { "dishId": "dish_1560801041", "kcalApeDeltaPp": 47.69872445941521, "pipelineDeltaMs": -130 }, { "dishId": "dish_1563566965", "kcalApeDeltaPp": -6.5804335415401916, "pipelineDeltaMs": -76 }, { "dishId": "dish_1565896745", "kcalApeDeltaPp": -20.933936068611253, "pipelineDeltaMs": 222 }, { "dishId": "dish_1564689108", "kcalApeDeltaPp": 9.98465794825416, "pipelineDeltaMs": 358 } ] }, { "model": "google/gemini-3.8-flash", "baseline": "google/gemini-3.1-flash-lite", "pairedPlates": 47, "kcalMeanApeDeltaPp": -8.617815673116697, "plateBootstrap95Pp": [ -20.764382674252516, 1.2668088021155706 ], "medianPairedPipelineDeltaMs": 4287.5, "qualifies": false, "plateDiffs": [ { "dishId": "dish_1566414291", "kcalApeDeltaPp": 6.895710091831159, "pipelineDeltaMs": 524 }, { "dishId": "dish_1562776497", "kcalApeDeltaPp": -12.729906828938127, "pipelineDeltaMs": 1850.5 }, { "dishId": "dish_1560801883", "kcalApeDeltaPp": -13.095539450413984, "pipelineDeltaMs": 5996 }, { "dishId": "dish_1563221157", "kcalApeDeltaPp": 19.31713789593951, "pipelineDeltaMs": 5468.5 }, { "dishId": "dish_1557862783", "kcalApeDeltaPp": -21.386980061717463, "pipelineDeltaMs": 2312.5 }, { "dishId": "dish_1565894910", "kcalApeDeltaPp": -15.201641810083267, "pipelineDeltaMs": 8317.5 }, { "dishId": "dish_1564773976", "kcalApeDeltaPp": 24.909237928698875, "pipelineDeltaMs": 3173.5 }, { "dishId": "dish_1561148042", "kcalApeDeltaPp": -22.798045729672786, "pipelineDeltaMs": 4845.5 }, { "dishId": "dish_1564686966", "kcalApeDeltaPp": -34.03636051706692, "pipelineDeltaMs": 1199 }, { "dishId": "dish_1562617703", "kcalApeDeltaPp": 8.709566645060212, "pipelineDeltaMs": 5209.5 }, { "dishId": "dish_1565711570", "kcalApeDeltaPp": -13.211559634907577, "pipelineDeltaMs": 1807.5 }, { "dishId": "dish_1558373159", "kcalApeDeltaPp": 8.66345518629598, "pipelineDeltaMs": 2187 }, { "dishId": "dish_1561577848", "kcalApeDeltaPp": 9.899830861593664, "pipelineDeltaMs": 3441 }, { "dishId": "dish_1560368570", "kcalApeDeltaPp": -15.1996804778932, "pipelineDeltaMs": 14783.5 }, { "dishId": "dish_1566587143", "kcalApeDeltaPp": 3.371744840063167, "pipelineDeltaMs": 7643 }, { "dishId": "dish_1561405411", "kcalApeDeltaPp": -4.619140032675126, "pipelineDeltaMs": 4287.5 }, { "dishId": "dish_1566316757", "kcalApeDeltaPp": 31.892386683796495, "pipelineDeltaMs": 1027.5 }, { "dishId": "dish_1564588919", "kcalApeDeltaPp": 30.43660380927031, "pipelineDeltaMs": 8687 }, { "dishId": "dish_1562876531", "kcalApeDeltaPp": -74.36398772910952, "pipelineDeltaMs": 1521.5 }, { "dishId": "dish_1558114284", "kcalApeDeltaPp": -16.803302811833017, "pipelineDeltaMs": 1767 }, { "dishId": "dish_1565638599", "kcalApeDeltaPp": -2.367320512439262, "pipelineDeltaMs": 7177.5 }, { "dishId": "dish_1561662216", "kcalApeDeltaPp": -7.3804594709033005, "pipelineDeltaMs": 14304 }, { "dishId": "dish_1565030350", "kcalApeDeltaPp": 4.457889980329838, "pipelineDeltaMs": 1127 }, { "dishId": "dish_1558113941", "kcalApeDeltaPp": 15.78193864148648, "pipelineDeltaMs": 2325.5 }, { "dishId": "dish_1561663606", "kcalApeDeltaPp": 4.390226618129496, "pipelineDeltaMs": 4505 }, { "dishId": "dish_1559850338", "kcalApeDeltaPp": 37.771926167917435, "pipelineDeltaMs": 4345.5 }, { "dishId": "dish_1565898128", "kcalApeDeltaPp": 2.5805952905630303, "pipelineDeltaMs": 1398.5 }, { "dishId": "dish_1565118782", "kcalApeDeltaPp": 15.887276662781034, "pipelineDeltaMs": 1812.5 }, { "dishId": "dish_1565898158", "kcalApeDeltaPp": 6.550566623788981, "pipelineDeltaMs": 2196 }, { "dishId": "dish_1560800951", "kcalApeDeltaPp": -208.60429321887554, "pipelineDeltaMs": 7412.5 }, { "dishId": "dish_1561739265", "kcalApeDeltaPp": -16.24947970741912, "pipelineDeltaMs": 4638.5 }, { "dishId": "dish_1562774206", "kcalApeDeltaPp": -5.890946201435873, "pipelineDeltaMs": 2931 }, { "dishId": "dish_1563295044", "kcalApeDeltaPp": -51.732537474929465, "pipelineDeltaMs": 4246.5 }, { "dishId": "dish_1559239369", "kcalApeDeltaPp": 22.27314424493041, "pipelineDeltaMs": 4892 }, { "dishId": "dish_1566502573", "kcalApeDeltaPp": -2.720599231810141, "pipelineDeltaMs": 5711 }, { "dishId": "dish_1565123881", "kcalApeDeltaPp": -0.9525056191256165, "pipelineDeltaMs": 5686 }, { "dishId": "dish_1558461792", "kcalApeDeltaPp": 1.7643504531722067, "pipelineDeltaMs": 9901 }, { "dishId": "dish_1560203424", "kcalApeDeltaPp": 19.955551118332316, "pipelineDeltaMs": 5025.5 }, { "dishId": "dish_1562099157", "kcalApeDeltaPp": -97.04756448183245, "pipelineDeltaMs": 1508 }, { "dishId": "dish_1565898230", "kcalApeDeltaPp": -3.906039587751607, "pipelineDeltaMs": 6154 }, { "dishId": "dish_1561662814", "kcalApeDeltaPp": -43.41638347290852, "pipelineDeltaMs": 1008 }, { "dishId": "dish_1563984296", "kcalApeDeltaPp": -26.13251878507037, "pipelineDeltaMs": 2801.5 }, { "dishId": "dish_1567785778", "kcalApeDeltaPp": -1.0213915400781168, "pipelineDeltaMs": 4605 }, { "dishId": "dish_1560801041", "kcalApeDeltaPp": 36.94670205489807, "pipelineDeltaMs": 3311.5 }, { "dishId": "dish_1563566965", "kcalApeDeltaPp": -3.186667953376187, "pipelineDeltaMs": 10924.5 }, { "dishId": "dish_1565896745", "kcalApeDeltaPp": -9.772335807012759, "pipelineDeltaMs": 2504.5 }, { "dishId": "dish_1564689108", "kcalApeDeltaPp": 6.33400971391584, "pipelineDeltaMs": 5351 } ] } ], "spend": { "at": "2026-09-18T10:42:51.839Z", "confirmedUsd": 0.6024330000000001, "reservedUsd": 0, "capUsd": 1, "calls": 388 }, "retrieval": { "queries": 395, "fresh": 383, "p50Ms": 850, "p95Ms": 993.3999999999999 }, "experimentCompleted": "2026-09-18", "published": "2026-10-06", "dataset": "Nutrition5k", "datasetUrl": "https://github.com/google-research-datasets/Nutrition5k", "datasetLicense": "https://creativecommons.org/licenses/by/4.0/" }