{
 "page": "https://nrlc.ai/en-us/insights/page-anatomy-for-ai-answers/",
 "questions": "/assets/data/page-anatomy-test/questions.json",
 "tested": "2026-11-09",
 "method": {
  "whole_page": "Visible text of the page's main content from the server-sent HTML, no JavaScript.",
  "passages": "Main-content text split into passages of about 200 words (whole paragraphs); the 2 best matches per question by BM25.",
  "prompt": "Answer the question using only the text below. If the answer is not in the text, reply exactly: not stated.\n\nTEXT:\n{{input}}\n\nQUESTION:\n{{question}}",
  "qwen": "qwen2.5:7b-instruct via Ollama, temperature 0, seed 0, one run.",
  "claude": "Claude Sonnet (claude-sonnet-5-5) via headless Claude Code, no tools, one turn, a fresh call per question, three runs.",
  "scoring": "Correct if the answer contains a gold answer after normalizing case, accents, dashes and thousands separators; traps are correct only if the answer is 'not stated'."
 },
 "before": {
  "qwen": {
   "label": "Live page before the rewrite",
   "model": "qwen2.5:7b-instruct",
   "input_words_whole_page": 786,
   "passages": 4,
   "answers": [
    {
     "id": "q01",
     "cond": "whole",
     "answer": "The answer passage at the top of the page should be about 200 words.",
     "correct": true
    },
    {
     "id": "q01",
     "cond": "passages",
     "answer": "The answer passage at the top of the page should be about 200 words.",
     "correct": true
    },
    {
     "id": "q02",
     "cond": "whole",
     "answer": "The 4 best-matching passages answered more questions correctly than the whole article, using 89% fewer tokens.",
     "correct": false
    },
    {
     "id": "q02",
     "cond": "passages",
     "answer": "The 4 best-matching passages answered more questions correctly than the whole article, using 89% fewer tokens.",
     "correct": false
    },
    {
     "id": "q03",
     "cond": "whole",
     "answer": "89%",
     "correct": true
    },
    {
     "id": "q03",
     "cond": "passages",
     "answer": "89% fewer tokens.",
     "correct": true
    },
    {
     "id": "q04",
     "cond": "whole",
     "answer": "16%",
     "correct": true
    },
    {
     "id": "q04",
     "cond": "passages",
     "answer": "128 of 800",
     "correct": true
    },
    {
     "id": "q05",
     "cond": "whole",
     "answer": "Model-written summaries kept the answer for 16% of questions, and answering from the summary scored 9.4%.",
     "correct": true
    },
    {
     "id": "q05",
     "cond": "passages",
     "answer": "Model-written summaries kept the answer for 16% of questions, and answering from the summary scored 9.4%.",
     "correct": true
    },
    {
     "id": "q06",
     "cond": "whole",
     "answer": "Rewriting every article into one-line facts made it only 6% smaller and less accurate than the original prose.",
     "correct": true
    },
    {
     "id": "q06",
     "cond": "passages",
     "answer": "Rewriting every article into one-line facts made it only 6% smaller and less accurate than the original prose.",
     "correct": true
    },
    {
     "id": "q07",
     "cond": "whole",
     "answer": "-4.9 points",
     "correct": true
    },
    {
     "id": "q07",
     "cond": "passages",
     "answer": "-4.9 points",
     "correct": true
    },
    {
     "id": "q08",
     "cond": "whole",
     "answer": "4.9%",
     "correct": true
    },
    {
     "id": "q08",
     "cond": "passages",
     "answer": "4.9%",
     "correct": true
    },
    {
     "id": "q09",
     "cond": "whole",
     "answer": "Ten buyer questions take minutes.",
     "correct": true
    },
    {
     "id": "q09",
     "cond": "passages",
     "answer": "Start with the questions. List the ten things a buyer asks before they call: price, timing, what's included, where you work, what goes wrong.",
     "correct": true
    },
    {
     "id": "q10",
     "cond": "whole",
     "answer": "1,000 questions",
     "correct": true
    },
    {
     "id": "q10",
     "cond": "passages",
     "answer": "Ten questions were in the benchmark behind these numbers.",
     "correct": false
    },
    {
     "id": "t01",
     "cond": "whole",
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t01",
     "cond": "passages",
     "answer": "Not stated.",
     "correct": true
    },
    {
     "id": "t02",
     "cond": "whole",
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t02",
     "cond": "passages",
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t03",
     "cond": "whole",
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t03",
     "cond": "passages",
     "answer": "not stated",
     "correct": true
    }
   ]
  },
  "claude": {
   "label": "Live page before the rewrite",
   "model": "claude sonnet (claude -p, no tools)",
   "input_words_whole_page": 785,
   "passages": 4,
   "answers": [
    {
     "id": "q01",
     "cond": "whole",
     "run": 0,
     "answer": "About 200 words, written so that it makes sense on its own.",
     "correct": true
    },
    {
     "id": "q01",
     "cond": "whole",
     "run": 1,
     "answer": "About 200 words, written so it makes sense on its own.",
     "correct": true
    },
    {
     "id": "q01",
     "cond": "whole",
     "run": 2,
     "answer": "About 200 words. The text says to lead with the answer \"in about 200 words that make sense on their own.\"",
     "correct": true
    },
    {
     "id": "q01",
     "cond": "passages",
     "run": 0,
     "answer": "About 200 words, written so it makes sense on its own.",
     "correct": true
    },
    {
     "id": "q01",
     "cond": "passages",
     "run": 1,
     "answer": "About 200 words, written so it makes sense on its own.",
     "correct": true
    },
    {
     "id": "q01",
     "cond": "passages",
     "run": 2,
     "answer": "About 200 words, written so it makes sense on its own.",
     "correct": true
    },
    {
     "id": "q02",
     "cond": "whole",
     "run": 0,
     "answer": "The 4 best-matching passages of about 200 words answered more questions correctly than the whole article: 98.5% versus 95.1%, using 89% fewer tokens.",
     "correct": true
    },
    {
     "id": "q02",
     "cond": "whole",
     "run": 1,
     "answer": "The 4 best-matching passages answered 98.5% of questions correctly, compared with 95.1% for the whole article. They also used 89% fewer tokens.",
     "correct": true
    },
    {
     "id": "q02",
     "cond": "whole",
     "run": 2,
     "answer": "The 4 best-matching passages (about 200 words each) answered more questions correctly than the whole article: 98.5% versus 95.1%. They also used 89% fewer tokens.",
     "correct": true
    },
    {
     "id": "q02",
     "cond": "passages",
     "run": 0,
     "answer": "The 4 best-matching passages of about 200 words answered 98.5% of questions correctly, compared with 95.1% for the whole article. They also used 89% fewer tokens.",
     "correct": true
    },
    {
     "id": "q02",
     "cond": "passages",
     "run": 1,
     "answer": "The 4 best-matching passages (about 200 words each) answered 98.5% of questions correctly, compared with 95.1% for the whole article. They also used 89% fewer tokens.",
     "correct": true
    },
    {
     "id": "q02",
     "cond": "passages",
     "run": 2,
     "answer": "The 4 best-matching passages (about 200 words each) answered questions more accurately than the whole article: 98.5% versus 95.1%. They also used 89% fewer tokens.",
     "correct": true
    },
    {
     "id": "q03",
     "cond": "whole",
     "run": 0,
     "answer": "They used 89% fewer tokens than the whole article.",
     "correct": true
    },
    {
     "id": "q03",
     "cond": "whole",
     "run": 1,
     "answer": "89% fewer tokens.",
     "correct": true
    },
    {
     "id": "q03",
     "cond": "whole",
     "run": 2,
     "answer": "89% fewer tokens.",
     "correct": true
    },
    {
     "id": "q03",
     "cond": "passages",
     "run": 0,
     "answer": "89% fewer tokens.",
     "correct": true
    },
    {
     "id": "q03",
     "cond": "passages",
     "run": 1,
     "answer": "89% fewer tokens.",
     "correct": true
    },
    {
     "id": "q03",
     "cond": "passages",
     "run": 2,
     "answer": "The 4 best-matching passages used 89% fewer tokens than the whole article.",
     "correct": true
    },
    {
     "id": "q04",
     "cond": "whole",
     "run": 0,
     "answer": "128 of 800 questions (16%).",
     "correct": true
    },
    {
     "id": "q04",
     "cond": "whole",
     "run": 1,
     "answer": "128 of 800 questions (16%).",
     "correct": true
    },
    {
     "id": "q04",
     "cond": "whole",
     "run": 2,
     "answer": "128 questions (out of 800, about 16%).",
     "correct": true
    },
    {
     "id": "q04",
     "cond": "passages",
     "run": 0,
     "answer": "128 of 800 questions.",
     "correct": true
    },
    {
     "id": "q04",
     "cond": "passages",
     "run": 1,
     "answer": "128 of 800 questions.",
     "correct": true
    },
    {
     "id": "q04",
     "cond": "passages",
     "run": 2,
     "answer": "128 of 800 questions (16%).",
     "correct": true
    },
    {
     "id": "q05",
     "cond": "whole",
     "run": 0,
     "answer": "Answering from the summary scored 9.4%. (Model-written summaries also kept the answer for only 16% of questions, 128 of 800.)",
     "correct": true
    },
    {
     "id": "q05",
     "cond": "whole",
     "run": 1,
     "answer": "Answering from the summary scored 9.4%.",
     "correct": true
    },
    {
     "id": "q05",
     "cond": "whole",
     "run": 2,
     "answer": "Answering from the summary scored 9.4%.",
     "correct": true
    },
    {
     "id": "q05",
     "cond": "passages",
     "run": 0,
     "answer": "Answering from the summary scored 9.4%. (Model-written summaries also kept the answer for only 16% of questions, 128 of 800.)",
     "correct": true
    },
    {
     "id": "q05",
     "cond": "passages",
     "run": 1,
     "answer": "Answering from the summary scored 9.4%.",
     "correct": true
    },
    {
     "id": "q05",
     "cond": "passages",
     "run": 2,
     "answer": "Answering from the summary scored 9.4%.",
     "correct": true
    },
    {
     "id": "q06",
     "cond": "whole",
     "run": 0,
     "answer": "Rewriting every article into one-line facts made it only 6% smaller (and 4.9 points less accurate than the original prose).",
     "correct": true
    },
    {
     "id": "q06",
     "cond": "whole",
     "run": 1,
     "answer": "6% smaller.",
     "correct": true
    },
    {
     "id": "q06",
     "cond": "whole",
     "run": 2,
     "answer": "6% smaller",
     "correct": true
    },
    {
     "id": "q06",
     "cond": "passages",
     "run": 0,
     "answer": "Rewriting every article into one-line facts made it only 6% smaller (and 4.9 points less accurate than the original prose).",
     "correct": true
    },
    {
     "id": "q06",
     "cond": "passages",
     "run": 1,
     "answer": "The rewrite was only 6% smaller than the original article (and 4.9 points less accurate).",
     "correct": true
    },
    {
     "id": "q06",
     "cond": "passages",
     "run": 2,
     "answer": "The rewrite was only 6% smaller than the original article.",
     "correct": true
    },
    {
     "id": "q07",
     "cond": "whole",
     "run": 0,
     "answer": "4.9 points. The text says rewriting every article into one-line facts made it \"only 6% smaller and less accurate than the original prose\" (−6% size, −4.9 points).",
     "correct": true
    },
    {
     "id": "q07",
     "cond": "whole",
     "run": 1,
     "answer": "4.9 points",
     "correct": true
    },
    {
     "id": "q07",
     "cond": "whole",
     "run": 2,
     "answer": "4.9 points (the rewrite was also only 6% smaller than the original).",
     "correct": true
    },
    {
     "id": "q07",
     "cond": "passages",
     "run": 0,
     "answer": "4.9 points (the rewrite was 6% smaller but 4.9 points less accurate than the original prose).",
     "correct": true
    },
    {
     "id": "q07",
     "cond": "passages",
     "run": 1,
     "answer": "4.9 points (the rewrite was 6% smaller and 4.9 points less accurate than the original prose).",
     "correct": true
    },
    {
     "id": "q07",
     "cond": "passages",
     "run": 2,
     "answer": "4.9 points. The text says the rewrite into one-line facts made the articles \"only 6% smaller and less accurate than the original prose\" (−6% size, −4.9 points).",
     "correct": true
    },
    {
     "id": "q08",
     "cond": "whole",
     "run": 0,
     "answer": "The median share of a page's HTML that was visible text was 4.9%, based on 12 pages sampled.",
     "correct": true
    },
    {
     "id": "q08",
     "cond": "whole",
     "run": 1,
     "answer": "The median share of a page's HTML that was visible text was 4.9%, based on 12 pages sampled.",
     "correct": true
    },
    {
     "id": "q08",
     "cond": "whole",
     "run": 2,
     "answer": "The median share of a page's HTML that was visible text was 4.9%, based on 12 pages sampled.",
     "correct": true
    },
    {
     "id": "q08",
     "cond": "passages",
     "run": 0,
     "answer": "The median share of a page's HTML that was visible text was 4.9%, based on 12 pages sampled.",
     "correct": true
    },
    {
     "id": "q08",
     "cond": "passages",
     "run": 1,
     "answer": "The median share of a page's HTML that was visible text was 4.9%, based on 12 pages sampled.",
     "correct": true
    },
    {
     "id": "q08",
     "cond": "passages",
     "run": 2,
     "answer": "The median share of a page's HTML that was visible text was 4.9%, based on 12 sampled pages.",
     "correct": true
    },
    {
     "id": "q09",
     "cond": "whole",
     "run": 0,
     "answer": "Ten. The text says to list the ten things a buyer asks before they call (price, timing, what's included, where you work, what goes wrong) and to ask a model those ten questions with the page as its only source.",
     "correct": true
    },
    {
     "id": "q09",
     "cond": "whole",
     "run": 1,
     "answer": "Ten. The text says to list the ten things a buyer asks before they call, and notes that testing ten buyer questions takes minutes.",
     "correct": true
    },
    {
     "id": "q09",
     "cond": "whole",
     "run": 2,
     "answer": "Ten. The text says to list the ten things a buyer asks before they call, then give a model the page and ask those ten questions. It also notes that \"ten buyer questions take minutes.\"",
     "correct": true
    },
    {
     "id": "q09",
     "cond": "passages",
     "run": 0,
     "answer": "not stated",
     "correct": false
    },
    {
     "id": "q09",
     "cond": "passages",
     "run": 1,
     "answer": "not stated",
     "correct": false
    },
    {
     "id": "q09",
     "cond": "passages",
     "run": 2,
     "answer": "not stated.",
     "correct": false
    },
    {
     "id": "q10",
     "cond": "whole",
     "run": 0,
     "answer": "1,000 questions (asked seven ways, yielding 7,000 answers).",
     "correct": true
    },
    {
     "id": "q10",
     "cond": "whole",
     "run": 1,
     "answer": "1,000 questions (asked seven ways, giving 7,000 answers in total).",
     "correct": true
    },
    {
     "id": "q10",
     "cond": "whole",
     "run": 2,
     "answer": "1,000 questions (asked seven ways, for 7,000 answers in total).",
     "correct": true
    },
    {
     "id": "q10",
     "cond": "passages",
     "run": 0,
     "answer": "not stated",
     "correct": false
    },
    {
     "id": "q10",
     "cond": "passages",
     "run": 1,
     "answer": "not stated",
     "correct": false
    },
    {
     "id": "q10",
     "cond": "passages",
     "run": 2,
     "answer": "not stated",
     "correct": false
    },
    {
     "id": "t01",
     "cond": "whole",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t01",
     "cond": "whole",
     "run": 1,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t01",
     "cond": "whole",
     "run": 2,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t01",
     "cond": "passages",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t01",
     "cond": "passages",
     "run": 1,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t01",
     "cond": "passages",
     "run": 2,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t02",
     "cond": "whole",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t02",
     "cond": "whole",
     "run": 1,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t02",
     "cond": "whole",
     "run": 2,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t02",
     "cond": "passages",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t02",
     "cond": "passages",
     "run": 1,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t02",
     "cond": "passages",
     "run": 2,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t03",
     "cond": "whole",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t03",
     "cond": "whole",
     "run": 1,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t03",
     "cond": "whole",
     "run": 2,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t03",
     "cond": "passages",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t03",
     "cond": "passages",
     "run": 1,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t03",
     "cond": "passages",
     "run": 2,
     "answer": "not stated",
     "correct": true
    }
   ]
  },
  "ingest": {
   "specific_sentences_share": 0.358,
   "visible_text_share_of_html": 0.079,
   "sentences": 53,
   "fact_sentences": 19,
   "html_chars": 47680
  }
 },
 "after": {
  "qwen": {
   "label": "Rewritten page, final",
   "model": "qwen2.5:7b-instruct, temperature 0, seed 0",
   "input_words_whole_page": 1765,
   "passages": 8,
   "answers": [
    {
     "id": "q01",
     "cond": "whole",
     "run": 0,
     "answer": "The answer passage at the top of a page should run 150 to 250 words.",
     "correct": true
    },
    {
     "id": "q01",
     "cond": "passages",
     "run": 0,
     "answer": "The answer passage at the top of a page should run 150 to 250 words.",
     "correct": true
    },
    {
     "id": "q02",
     "cond": "whole",
     "run": 0,
     "answer": "The model answered 98.5% of the answerable questions correctly from the 4 best passages, compared to 95.1% when it read the whole article.",
     "correct": true
    },
    {
     "id": "q02",
     "cond": "passages",
     "run": 0,
     "answer": "The model answered 36 out of 39 questions correctly from the 4 best-matching passages when it read the whole article and got 39 answerable questions wrong.",
     "correct": false
    },
    {
     "id": "q03",
     "cond": "whole",
     "run": 0,
     "answer": "The 4 best-matching passages used 89% fewer input tokens than the whole article.",
     "correct": true
    },
    {
     "id": "q03",
     "cond": "passages",
     "run": 0,
     "answer": "The 4 best-matching passages used 89% fewer input tokens than the whole article.",
     "correct": true
    },
    {
     "id": "q04",
     "cond": "whole",
     "run": 0,
     "answer": "In NRLC's benchmark, the model's own summaries kept the answer for only 128 of the 800 answerable questions.",
     "correct": true
    },
    {
     "id": "q04",
     "cond": "passages",
     "run": 0,
     "answer": "A summary or key-takeaways box helps people skim, but it will not carry a page's facts. In NRLC's benchmark, the model's own summaries kept the answer for only 128 of the 800 answerable questions.",
     "correct": true
    },
    {
     "id": "q05",
     "cond": "whole",
     "run": 0,
     "answer": "The model's answers from the summaries were correct 9.4% of the time.",
     "correct": true
    },
    {
     "id": "q05",
     "cond": "passages",
     "run": 0,
     "answer": "Answers from the summaries were correct 9.4% of the time.",
     "correct": true
    },
    {
     "id": "q06",
     "cond": "whole",
     "run": 0,
     "answer": "The full fact-list rewrite of each article was only 6% smaller than the original prose.",
     "correct": true
    },
    {
     "id": "q06",
     "cond": "passages",
     "run": 0,
     "answer": "The rewrite into one-line facts was only 6% smaller than the original prose.",
     "correct": true
    },
    {
     "id": "q07",
     "cond": "whole",
     "run": 0,
     "answer": "The one-line-fact rewrite was 4.9 points less accurate than answers from the whole article, according to the text.",
     "correct": true
    },
    {
     "id": "q07",
     "cond": "passages",
     "run": 0,
     "answer": "The one-line-fact rewrite was 4.9 points less accurate than the answers from the whole article.",
     "correct": true
    },
    {
     "id": "q08",
     "cond": "whole",
     "run": 0,
     "answer": "On most web pages, visible text is a small share of the HTML. On 12 pages sampled in the first run of NRLC's benchmark, visible text was a median 4.9% of the HTML by characters.",
     "correct": true
    },
    {
     "id": "q08",
     "cond": "passages",
     "run": 0,
     "answer": "Visible text is a median 4.9% of the HTML by characters on most web pages.",
     "correct": true
    },
    {
     "id": "q09",
     "cond": "whole",
     "run": 0,
     "answer": "Test an answer page with 10 buyer questions plus 3 trap questions that the page does not answer.",
     "correct": true
    },
    {
     "id": "q09",
     "cond": "passages",
     "run": 0,
     "answer": "Test an answer page with 10 buyer questions plus 3 trap questions that the page does not answer.",
     "correct": true
    },
    {
     "id": "q10",
     "cond": "whole",
     "run": 0,
     "answer": "The benchmark behind these numbers had 1,000 questions in total.",
     "correct": true
    },
    {
     "id": "q10",
     "cond": "passages",
     "run": 0,
     "answer": "1,000 questions in total.",
     "correct": true
    },
    {
     "id": "t01",
     "cond": "whole",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t01",
     "cond": "passages",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t02",
     "cond": "whole",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t02",
     "cond": "passages",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t03",
     "cond": "whole",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t03",
     "cond": "passages",
     "run": 0,
     "answer": "not stated",
     "correct": true
    }
   ]
  },
  "claude": {
   "label": "Rewritten page, final",
   "model": "claude sonnet (claude -p, no tools)",
   "input_words_whole_page": 1765,
   "passages": 8,
   "answers": [
    {
     "id": "q01",
     "cond": "whole",
     "run": 0,
     "answer": "The answer passage at the top of the page should run 150 to 250 words. That length is long enough to carry the specifics and short enough to be retrieved and read as one passage; the passages in NRLC's benchmark were about 200 words each.",
     "correct": true
    },
    {
     "id": "q01",
     "cond": "whole",
     "run": 1,
     "answer": "The answer passage at the top of the page should run 150 to 250 words. That length is long enough to carry the specifics and short enough to be retrieved and read as one passage; the passages in NRLC's benchmark were about 200 words each.",
     "correct": true
    },
    {
     "id": "q01",
     "cond": "whole",
     "run": 2,
     "answer": "The answer passage at the top of a page should run 150 to 250 words. That is long enough to carry the specifics and short enough to be retrieved and read as one passage; the passages in NRLC's benchmark were about 200 words each. It has to make sense as the only text a model sees, so it names the subject, gives the answer with its number, unit, place and date, and says where the figure comes from.",
     "correct": true
    },
    {
     "id": "q01",
     "cond": "passages",
     "run": 0,
     "answer": "150 to 250 words.",
     "correct": true
    },
    {
     "id": "q01",
     "cond": "passages",
     "run": 1,
     "answer": "The answer passage at the top of the page should run 150 to 250 words.",
     "correct": true
    },
    {
     "id": "q01",
     "cond": "passages",
     "run": 2,
     "answer": "150 to 250 words.",
     "correct": true
    },
    {
     "id": "q02",
     "cond": "whole",
     "run": 0,
     "answer": "Given only the 4 best-matching passages, the model answered 98.5% of the answerable questions correctly, compared with 95.1% when it read the whole article. That is 3.4 percentage points higher for the passages.",
     "correct": true
    },
    {
     "id": "q02",
     "cond": "whole",
     "run": 1,
     "answer": "Given only the 4 best-matching passages of an article, the model answered 98.5% of the answerable questions correctly, compared with 95.1% when it read the whole article. The 4 passages also used 89% fewer input tokens than the whole article.",
     "correct": true
    },
    {
     "id": "q02",
     "cond": "whole",
     "run": 2,
     "answer": "The 4 best-matching passages answered 98.5% of the answerable questions correctly, compared with 95.1% when the model read the whole article. That is 3.4 percentage points higher, and the passages used 89% fewer input tokens.",
     "correct": true
    },
    {
     "id": "q02",
     "cond": "passages",
     "run": 0,
     "answer": "On the 39 answerable questions the model got wrong when it read the whole article, it answered 36 correctly from the 4 best-matching passages.",
     "correct": false
    },
    {
     "id": "q02",
     "cond": "passages",
     "run": 1,
     "answer": "On the 39 answerable questions the model got wrong when it read the whole article, it answered 36 correctly from the 4 best-matching passages.",
     "correct": false
    },
    {
     "id": "q02",
     "cond": "passages",
     "run": 2,
     "answer": "not stated",
     "correct": false
    },
    {
     "id": "q03",
     "cond": "whole",
     "run": 0,
     "answer": "The 4 best-matching passages used 89% fewer input tokens than the whole article.",
     "correct": true
    },
    {
     "id": "q03",
     "cond": "whole",
     "run": 1,
     "answer": "The 4 best-matching passages used 89% fewer input tokens than the whole article.",
     "correct": true
    },
    {
     "id": "q03",
     "cond": "whole",
     "run": 2,
     "answer": "The 4 best-matching passages used 89% fewer input tokens than the whole article.",
     "correct": true
    },
    {
     "id": "q03",
     "cond": "passages",
     "run": 0,
     "answer": "The 4 best-matching passages used 89% fewer input tokens than the whole article.",
     "correct": true
    },
    {
     "id": "q03",
     "cond": "passages",
     "run": 1,
     "answer": "89% fewer input tokens.",
     "correct": true
    },
    {
     "id": "q03",
     "cond": "passages",
     "run": 2,
     "answer": "89% fewer input tokens.",
     "correct": true
    },
    {
     "id": "q04",
     "cond": "whole",
     "run": 0,
     "answer": "A model-written summary kept the answer for 128 of the 800 answerable questions.",
     "correct": true
    },
    {
     "id": "q04",
     "cond": "whole",
     "run": 1,
     "answer": "A model-written summary kept the answer for 128 of the 800 answerable questions.",
     "correct": true
    },
    {
     "id": "q04",
     "cond": "whole",
     "run": 2,
     "answer": "A model-written summary kept the answer for 128 of the 800 answerable questions.",
     "correct": true
    },
    {
     "id": "q04",
     "cond": "passages",
     "run": 0,
     "answer": "128 of the 800 answerable questions.",
     "correct": true
    },
    {
     "id": "q04",
     "cond": "passages",
     "run": 1,
     "answer": "128 of the 800 answerable questions.",
     "correct": true
    },
    {
     "id": "q04",
     "cond": "passages",
     "run": 2,
     "answer": "128 of the 800 answerable questions.",
     "correct": true
    },
    {
     "id": "q05",
     "cond": "whole",
     "run": 0,
     "answer": "Answers from the model's summaries were correct 9.4% of the time in NRLC's benchmark.",
     "correct": true
    },
    {
     "id": "q05",
     "cond": "whole",
     "run": 1,
     "answer": "Answers from the summaries were correct 9.4% of the time. This is from NRLC's benchmark, where the model's own summaries kept the answer for only 128 of the 800 answerable questions. The summaries ran 122 to 493 words.",
     "correct": true
    },
    {
     "id": "q05",
     "cond": "whole",
     "run": 2,
     "answer": "Answers from the model's summaries were correct 9.4% of the time. The summaries kept the answer for only 128 of the 800 answerable questions and ran 122 to 493 words.",
     "correct": true
    },
    {
     "id": "q05",
     "cond": "passages",
     "run": 0,
     "answer": "Answers from the summaries were correct 9.4% of the time.",
     "correct": true
    },
    {
     "id": "q05",
     "cond": "passages",
     "run": 1,
     "answer": "Answers from the summaries were correct 9.4% of the time.",
     "correct": true
    },
    {
     "id": "q05",
     "cond": "passages",
     "run": 2,
     "answer": "Answers from the summaries were correct 9.4% of the time. The summaries kept the answer for only 128 of the 800 answerable questions.",
     "correct": true
    },
    {
     "id": "q06",
     "cond": "whole",
     "run": 0,
     "answer": "The full fact-list rewrite of each article was only 6% smaller than the original prose.",
     "correct": true
    },
    {
     "id": "q06",
     "cond": "whole",
     "run": 1,
     "answer": "The full fact-list rewrite of each article was only 6% smaller than the original prose.",
     "correct": true
    },
    {
     "id": "q06",
     "cond": "whole",
     "run": 2,
     "answer": "The full fact-list rewrite was only 6% smaller than the original prose.",
     "correct": true
    },
    {
     "id": "q06",
     "cond": "passages",
     "run": 0,
     "answer": "6% smaller.",
     "correct": true
    },
    {
     "id": "q06",
     "cond": "passages",
     "run": 1,
     "answer": "6% smaller than the original prose.",
     "correct": true
    },
    {
     "id": "q06",
     "cond": "passages",
     "run": 2,
     "answer": "6% smaller.",
     "correct": true
    },
    {
     "id": "q07",
     "cond": "whole",
     "run": 0,
     "answer": "4.9 points. The text says answers from the full fact-list rewrite were 4.9 points less accurate than answers from the whole original article.",
     "correct": true
    },
    {
     "id": "q07",
     "cond": "whole",
     "run": 1,
     "answer": "4.9 points. Answers from the full fact-list rewrite were 4.9 points less accurate than answers from the whole article.",
     "correct": true
    },
    {
     "id": "q07",
     "cond": "whole",
     "run": 2,
     "answer": "4.9 points. Answers from the full fact-list rewrite were 4.9 points less accurate than answers from the whole original article.",
     "correct": true
    },
    {
     "id": "q07",
     "cond": "passages",
     "run": 0,
     "answer": "4.9 points",
     "correct": true
    },
    {
     "id": "q07",
     "cond": "passages",
     "run": 1,
     "answer": "4.9 points less accurate.",
     "correct": true
    },
    {
     "id": "q07",
     "cond": "passages",
     "run": 2,
     "answer": "4.9 points. Answers from the one-line-fact rewrite were 4.9 points less accurate than answers from the whole article.",
     "correct": true
    },
    {
     "id": "q08",
     "cond": "whole",
     "run": 0,
     "answer": "On 12 pages sampled in the first run of NRLC's benchmark (from Wikipedia, BBC News, Shopify, Stripe and others), visible text was a median 4.9% of the HTML by characters.",
     "correct": true
    },
    {
     "id": "q08",
     "cond": "whole",
     "run": 1,
     "answer": "On 12 pages sampled in the first run of NRLC's benchmark (from Wikipedia, BBC News, Shopify, Stripe and others), visible text was a median of 4.9% of the HTML by characters.",
     "correct": true
    },
    {
     "id": "q08",
     "cond": "whole",
     "run": 2,
     "answer": "On most web pages, visible text is a small share of the HTML. In the 12 pages sampled in the first run of NRLC's benchmark (from Wikipedia, BBC News, Shopify, Stripe and others), visible text was a median of 4.9% of the HTML by characters.",
     "correct": true
    },
    {
     "id": "q08",
     "cond": "passages",
     "run": 0,
     "answer": "Visible text was a median of 4.9% of the HTML by characters, based on 12 sampled pages (from Wikipedia, BBC News, Shopify, Stripe and others) in the first run of NRLC's benchmark.",
     "correct": true
    },
    {
     "id": "q08",
     "cond": "passages",
     "run": 1,
     "answer": "Visible text was a median 4.9% of the HTML by characters, based on 12 sampled pages in the first run of NRLC's benchmark.",
     "correct": true
    },
    {
     "id": "q08",
     "cond": "passages",
     "run": 2,
     "answer": "Visible text was a median of 4.9% of the HTML by characters, based on 12 sampled pages in the first run of NRLC's benchmark.",
     "correct": true
    },
    {
     "id": "q09",
     "cond": "whole",
     "run": 0,
     "answer": "Test a page with 10 buyer questions plus 3 trap questions that the page does not answer, 13 in total. A decision page, which helps a buyer choose between options, gets 6 buyer questions, 3 comparison questions, 2 recommendation questions, 1 question about where the other option is better, and 3 traps. Write the questions before drafting, save them with the date, and don't change them once testing starts.",
     "correct": true
    },
    {
     "id": "q09",
     "cond": "whole",
     "run": 1,
     "answer": "Test a page with 10 buyer questions plus 3 trap questions that the page does not answer, 13 in total. A decision page, which helps a buyer choose between options, gets 6 buyer questions, 3 comparison questions, 2 recommendation questions, 1 question about where the other option is better, and 3 traps.",
     "correct": true
    },
    {
     "id": "q09",
     "cond": "whole",
     "run": 2,
     "answer": "Test a page with 10 buyer questions plus 3 trap questions that the page does not answer, 13 in total. A decision page (one that helps a buyer choose between options) gets 6 buyer questions, 3 comparison questions, 2 recommendation questions, 1 question about where the other option is better, and 3 traps.",
     "correct": true
    },
    {
     "id": "q09",
     "cond": "passages",
     "run": 0,
     "answer": "Test an answer page with 10 buyer questions plus 3 trap questions that the page does not answer (13 total). A decision page gets 6 buyer questions, 3 comparison questions, 2 recommendation questions, 1 question about where the other option is better, plus 3 traps.",
     "correct": true
    },
    {
     "id": "q09",
     "cond": "passages",
     "run": 1,
     "answer": "Test an answer page with 10 buyer questions plus 3 trap questions that the page does not answer (13 total). For a decision page, use 6 buyer questions, 3 comparison questions, 2 recommendation questions, 1 question about where the other option is better, and 3 traps.",
     "correct": true
    },
    {
     "id": "q09",
     "cond": "passages",
     "run": 2,
     "answer": "Test an answer page with **10 buyer questions plus 3 trap questions** (13 in total) that the page does not answer.\n\nA decision page gets 6 buyer questions, 3 comparison questions, 2 recommendation questions, 1 question about where the other option is better, and 3 traps.",
     "correct": true
    },
    {
     "id": "q10",
     "cond": "whole",
     "run": 0,
     "answer": "The benchmark had 1,000 questions in total, of which 200 were deliberately unanswerable from the article.",
     "correct": true
    },
    {
     "id": "q10",
     "cond": "whole",
     "run": 1,
     "answer": "The benchmark had 1,000 questions in total, about 50 English Wikipedia articles. 200 of them were deliberately unanswerable from the article, which leaves 800 answerable questions.",
     "correct": true
    },
    {
     "id": "q10",
     "cond": "whole",
     "run": 2,
     "answer": "The benchmark had 1,000 questions in total, of which 200 were deliberately unanswerable from the article. The other 800 were answerable.",
     "correct": true
    },
    {
     "id": "q10",
     "cond": "passages",
     "run": 0,
     "answer": "There were 1,000 questions in the benchmark.",
     "correct": true
    },
    {
     "id": "q10",
     "cond": "passages",
     "run": 1,
     "answer": "The benchmark had 1,000 questions in total.",
     "correct": true
    },
    {
     "id": "q10",
     "cond": "passages",
     "run": 2,
     "answer": "The benchmark had 1,000 questions in total.",
     "correct": true
    },
    {
     "id": "t01",
     "cond": "whole",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t01",
     "cond": "whole",
     "run": 1,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t01",
     "cond": "whole",
     "run": 2,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t01",
     "cond": "passages",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t01",
     "cond": "passages",
     "run": 1,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t01",
     "cond": "passages",
     "run": 2,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t02",
     "cond": "whole",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t02",
     "cond": "whole",
     "run": 1,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t02",
     "cond": "whole",
     "run": 2,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t02",
     "cond": "passages",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t02",
     "cond": "passages",
     "run": 1,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t02",
     "cond": "passages",
     "run": 2,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t03",
     "cond": "whole",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t03",
     "cond": "whole",
     "run": 1,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t03",
     "cond": "whole",
     "run": 2,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t03",
     "cond": "passages",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t03",
     "cond": "passages",
     "run": 1,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t03",
     "cond": "passages",
     "run": 2,
     "answer": "not stated",
     "correct": true
    }
   ]
  },
  "ingest": {
   "specific_sentences_share": 0.664,
   "visible_text_share_of_html": 0.129,
   "sentences": 95,
   "fact_sentences": 65,
   "html_chars": 54886
  }
 },
 "drafts_that_failed": [
  {
   "label": "Rewrite, draft 1: qwen whole page 8 of 10",
   "model": "qwen2.5:7b-instruct, temperature 0, seed 0",
   "input_words_whole_page": 1704,
   "passages": 8,
   "answers": [
    {
     "id": "q01",
     "cond": "whole",
     "run": 0,
     "answer": "The answer passage at the top of a page should run 150 to 250 words.",
     "correct": true
    },
    {
     "id": "q01",
     "cond": "passages",
     "run": 0,
     "answer": "The answer passage at the top of a page should run 150 to 250 words.",
     "correct": true
    },
    {
     "id": "q02",
     "cond": "whole",
     "run": 0,
     "answer": "The model answered 98.5% of the answerable questions correctly from the 4 best passages, compared with 95.1% when it read the whole article. Additionally, the model used 89% fewer input tokens when using the 4 best passages.",
     "correct": true
    },
    {
     "id": "q02",
     "cond": "passages",
     "run": 0,
     "answer": "The model answered 36 out of 39 questions correctly from the 4 best-matching passages when it read the whole article, which contained 39 answerable questions, and got 39 answerable questions wrong when reading the whole article.",
     "correct": false
    },
    {
     "id": "q03",
     "cond": "whole",
     "run": 0,
     "answer": "not stated",
     "correct": false
    },
    {
     "id": "q03",
     "cond": "passages",
     "run": 0,
     "answer": "89% fewer input tokens.",
     "correct": true
    },
    {
     "id": "q04",
     "cond": "whole",
     "run": 0,
     "answer": "For 128 of the 800 answerable questions, a model-written summary kept the answer. However, answers from those summaries were correct only 9.4% of the time.",
     "correct": true
    },
    {
     "id": "q04",
     "cond": "passages",
     "run": 0,
     "answer": "For 128 out of 800 answerable questions, a model-written summary kept the answer.",
     "correct": true
    },
    {
     "id": "q05",
     "cond": "whole",
     "run": 0,
     "answer": "In NRLC's benchmark, summaries written by the same model still contained the answer for only 128 of the 800 answerable questions, and answers from those summaries were correct 9.4% of the time.",
     "correct": true
    },
    {
     "id": "q05",
     "cond": "passages",
     "run": 0,
     "answer": "Answers from the summaries were correct 9.4% of the time.",
     "correct": true
    },
    {
     "id": "q06",
     "cond": "whole",
     "run": 0,
     "answer": "The full fact-list rewrite of each article was only 6% smaller than the original prose.",
     "correct": true
    },
    {
     "id": "q06",
     "cond": "passages",
     "run": 0,
     "answer": "The rewrite into one-line facts was only 6% smaller than the original prose.",
     "correct": true
    },
    {
     "id": "q07",
     "cond": "whole",
     "run": 0,
     "answer": "The one-line-fact rewrite was 4.9 points less accurate than answers from the whole article, according to the text.",
     "correct": true
    },
    {
     "id": "q07",
     "cond": "passages",
     "run": 0,
     "answer": "4.9 points",
     "correct": true
    },
    {
     "id": "q08",
     "cond": "whole",
     "run": 0,
     "answer": "On most web pages, visible text is a small share of the HTML. On 12 pages sampled in the first run of NRLC's benchmark, visible text was a median 4.9% of the HTML by characters.",
     "correct": true
    },
    {
     "id": "q08",
     "cond": "passages",
     "run": 0,
     "answer": "Visible text is a median 4.9% of the HTML by characters on most web pages, according to the sampled pages.",
     "correct": true
    },
    {
     "id": "q09",
     "cond": "whole",
     "run": 0,
     "answer": "Test an answer page with 10 buyer questions plus 3 trap questions that the page does not answer.",
     "correct": true
    },
    {
     "id": "q09",
     "cond": "passages",
     "run": 0,
     "answer": "Test an answer page with 10 buyer questions plus 3 trap questions that the page does not answer.",
     "correct": true
    },
    {
     "id": "q10",
     "cond": "whole",
     "run": 0,
     "answer": "800 of the questions were answerable from the article and 200 were not.",
     "correct": false
    },
    {
     "id": "q10",
     "cond": "passages",
     "run": 0,
     "answer": "1,000",
     "correct": true
    },
    {
     "id": "t01",
     "cond": "whole",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t01",
     "cond": "passages",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t02",
     "cond": "whole",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t02",
     "cond": "passages",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t03",
     "cond": "whole",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t03",
     "cond": "passages",
     "run": 0,
     "answer": "not stated",
     "correct": true
    }
   ]
  },
  {
   "label": "Rewrite, draft 2: qwen 8 of 10 both ways",
   "model": "qwen2.5:7b-instruct, temperature 0, seed 0",
   "input_words_whole_page": 1711,
   "passages": 8,
   "answers": [
    {
     "id": "q01",
     "cond": "whole",
     "run": 0,
     "answer": "The answer passage at the top of a page should run 150 to 250 words.",
     "correct": true
    },
    {
     "id": "q01",
     "cond": "passages",
     "run": 0,
     "answer": "The answer passage at the top of a page should run 150 to 250 words.",
     "correct": true
    },
    {
     "id": "q02",
     "cond": "whole",
     "run": 0,
     "answer": "The model answered 98.5% of the answerable questions correctly from the 4 best passages, compared with 95.1% when it read the whole article.",
     "correct": true
    },
    {
     "id": "q02",
     "cond": "passages",
     "run": 0,
     "answer": "The model answered 36 out of 39 questions correctly from the 4 best-matching passages when it read the whole article and got 39 answerable questions wrong.",
     "correct": false
    },
    {
     "id": "q03",
     "cond": "whole",
     "run": 0,
     "answer": "The 4 best-matching passages used 89% fewer input tokens than the whole article.",
     "correct": true
    },
    {
     "id": "q03",
     "cond": "passages",
     "run": 0,
     "answer": "The 4 best-matching passages used 89% fewer input tokens than the whole article.",
     "correct": true
    },
    {
     "id": "q04",
     "cond": "whole",
     "run": 0,
     "answer": "For 128 of the 800 answerable questions, a model-written summary kept the answer. However, answers from those summaries were correct only 9.4% of the time.",
     "correct": true
    },
    {
     "id": "q04",
     "cond": "passages",
     "run": 0,
     "answer": "498",
     "correct": false
    },
    {
     "id": "q05",
     "cond": "whole",
     "run": 0,
     "answer": "not stated",
     "correct": false
    },
    {
     "id": "q05",
     "cond": "passages",
     "run": 0,
     "answer": "Answers from the summaries were correct 9.4% of the time.",
     "correct": true
    },
    {
     "id": "q06",
     "cond": "whole",
     "run": 0,
     "answer": "The full fact-list rewrite of each article was only 6% smaller than the original prose.",
     "correct": true
    },
    {
     "id": "q06",
     "cond": "passages",
     "run": 0,
     "answer": "The rewrite into one-line facts was only 6% smaller than the original prose.",
     "correct": true
    },
    {
     "id": "q07",
     "cond": "whole",
     "run": 0,
     "answer": "The one-line-fact rewrite was 4.9 points less accurate than the original prose.",
     "correct": true
    },
    {
     "id": "q07",
     "cond": "passages",
     "run": 0,
     "answer": "4.9 points",
     "correct": true
    },
    {
     "id": "q08",
     "cond": "whole",
     "run": 0,
     "answer": "On most web pages, visible text is a small share of the HTML. On 12 pages sampled in the first run of NRLC's benchmark, visible text was a median 4.9% of the HTML by characters.",
     "correct": true
    },
    {
     "id": "q08",
     "cond": "passages",
     "run": 0,
     "answer": "Visible text is a median 4.9% of the HTML by characters on most web pages, according to the sampled pages.",
     "correct": true
    },
    {
     "id": "q09",
     "cond": "whole",
     "run": 0,
     "answer": "Test an answer page with 10 buyer questions plus 3 trap questions that the page does not answer.",
     "correct": true
    },
    {
     "id": "q09",
     "cond": "passages",
     "run": 0,
     "answer": "Test an answer page with 10 buyer questions plus 3 trap questions that the page does not answer.",
     "correct": true
    },
    {
     "id": "q10",
     "cond": "whole",
     "run": 0,
     "answer": "800",
     "correct": false
    },
    {
     "id": "q10",
     "cond": "passages",
     "run": 0,
     "answer": "1,000",
     "correct": true
    },
    {
     "id": "t01",
     "cond": "whole",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t01",
     "cond": "passages",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t02",
     "cond": "whole",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t02",
     "cond": "passages",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t03",
     "cond": "whole",
     "run": 0,
     "answer": "not stated",
     "correct": true
    },
    {
     "id": "t03",
     "cond": "passages",
     "run": 0,
     "answer": "not stated",
     "correct": true
    }
   ]
  }
 ]
}