{
 "conditions": {
  "A": {
   "name": "Full text",
   "n": 100,
   "input_tokens_mean": 8585.15,
   "input_tokens_ci": [
    8451.75,
    8717.26
   ],
   "prefill_ms_mean": 1404.1599999999994,
   "accuracy": 0.975,
   "accuracy_ci": [
    0.9375,
    1.0
   ],
   "abstention": 1.0,
   "abstention_ci": [
    1.0,
    1.0
   ],
   "made_up_on_unanswerable": 0.0
  },
  "B2": {
   "name": "RAG, top 2 chunks",
   "n": 100,
   "input_tokens_mean": 515.89,
   "input_tokens_ci": [
    498.51,
    533.03
   ],
   "prefill_ms_mean": 922.1670000000001,
   "accuracy": 0.95,
   "accuracy_ci": [
    0.9,
    0.9875
   ],
   "abstention": 1.0,
   "abstention_ci": [
    1.0,
    1.0
   ],
   "made_up_on_unanswerable": 0.0,
   "accuracy_diff_vs_A": [
    -0.025,
    -0.0875,
    0.0375
   ],
   "token_reduction_vs_A": 0.939909028962802
  },
  "B4": {
   "name": "RAG, top 4 chunks",
   "n": 100,
   "input_tokens_mean": 963.19,
   "input_tokens_ci": [
    936.78,
    991.26
   ],
   "prefill_ms_mean": 2363.635,
   "accuracy": 0.9875,
   "accuracy_ci": [
    0.9625,
    1.0
   ],
   "abstention": 1.0,
   "abstention_ci": [
    1.0,
    1.0
   ],
   "made_up_on_unanswerable": 0.0,
   "accuracy_diff_vs_A": [
    0.0125,
    -0.025,
    0.0625
   ],
   "token_reduction_vs_A": 0.8878074349312476
  },
  "C": {
   "name": "Summary",
   "n": 100,
   "input_tokens_mean": 521.75,
   "input_tokens_ci": [
    494.62,
    548.35
   ],
   "prefill_ms_mean": 531.0169999999999,
   "accuracy": 0.1375,
   "accuracy_ci": [
    0.0625,
    0.2125
   ],
   "abstention": 1.0,
   "abstention_ci": [
    1.0,
    1.0
   ],
   "made_up_on_unanswerable": 0.0,
   "accuracy_diff_vs_A": [
    -0.8375,
    -0.9125,
    -0.75
   ],
   "token_reduction_vs_A": 0.9392264549833143,
   "prep_tokens_per_article": 9043.2,
   "prep_tokens_per_question": 904.32,
   "break_even_questions": 1.1215120172631894
  },
  "D10": {
   "name": "Croutons, top 10",
   "n": 100,
   "input_tokens_mean": 270.15,
   "input_tokens_ci": [
    260.46,
    280.35
   ],
   "prefill_ms_mean": 996.9929999999999,
   "accuracy": 0.925,
   "accuracy_ci": [
    0.8625,
    0.975
   ],
   "abstention": 0.95,
   "abstention_ci": [
    0.85,
    1.0
   ],
   "made_up_on_unanswerable": 0.050000000000000044,
   "accuracy_diff_vs_A": [
    -0.05,
    -0.125,
    0.0125
   ],
   "token_reduction_vs_A": 0.9685328736248057,
   "prep_tokens_per_article": 17654.6,
   "prep_tokens_per_question": 1765.4599999999998,
   "break_even_questions": 2.123223090799759
  },
  "D25": {
   "name": "Croutons, top 25",
   "n": 100,
   "input_tokens_mean": 593.59,
   "input_tokens_ci": [
    571.05,
    617.24
   ],
   "prefill_ms_mean": 1942.5149999999994,
   "accuracy": 0.9375,
   "accuracy_ci": [
    0.875,
    0.9875
   ],
   "abstention": 1.0,
   "abstention_ci": [
    1.0,
    1.0
   ],
   "made_up_on_unanswerable": 0.0,
   "accuracy_diff_vs_A": [
    -0.0375,
    -0.1,
    0.025
   ],
   "token_reduction_vs_A": 0.930858517323518,
   "prep_tokens_per_article": 17654.6,
   "prep_tokens_per_question": 1765.4599999999998,
   "break_even_questions": 2.2091556592204777
  },
  "Dall": {
   "name": "All croutons",
   "n": 100,
   "input_tokens_mean": 7915.65,
   "input_tokens_ci": [
    7770.51,
    8057.63
   ],
   "prefill_ms_mean": 3338.4480000000003,
   "accuracy": 0.925,
   "accuracy_ci": [
    0.8625,
    0.975
   ],
   "abstention": 1.0,
   "abstention_ci": [
    1.0,
    1.0
   ],
   "made_up_on_unanswerable": 0.0,
   "accuracy_diff_vs_A": [
    -0.05,
    -0.1125,
    0.0
   ],
   "token_reduction_vs_A": 0.07798349475547894,
   "prep_tokens_per_article": 17654.6,
   "prep_tokens_per_question": 1765.4599999999998,
   "break_even_questions": 26.369828230022403
  }
 },
 "n_questions": 100,
 "caching_on_A": {
  "Anthropic (cache read)": {
   "effective_input_tokens_after_first_question": 858.515,
   "reduction": 0.9
  },
  "OpenAI (cached input)": {
   "effective_input_tokens_after_first_question": 4292.575,
   "reduction": 0.5
  },
  "Google Gemini (implicit cache)": {
   "effective_input_tokens_after_first_question": 2146.2875,
   "reduction": 0.75
  }
 },
 "hypotheses": {
  "H1_token_reduction_ge_80pct": true,
  "H2_accuracy_within_5pts": false,
  "H3_made_up_le_B4": true
 }
}