FABLES: Evaluating faithfulness and content selection in book-length summarization

{
  "archived": false,
  "revision": {
    "ref": "v2 (revised 30 Sep 2024); comments: preprint - 39 pages",
    "pinned": true,
    "source_url": "https://arxiv.org/abs/2404.01261"
  },
  "checked_at": "2026-10-17T00:00:00.000Z",
  "usefulness": "Human annotation study (3,158 claims in LLM summaries of 26 books) finding that most unfaithful claims concern events and character states and that LLM raters correlate poorly with human judgments. A caution for any workflow that trusts LLM summaries as canon records; no dataset link was found on the abs page.",
  "source_type": "paper",
  "availability": "available",
  "source_owner": "Yekyung Kim, Yapei Chang, Marzena Karpinska, Aparna Garimella, Varun Manjunatha, Kyle Lo, Tanya Goyal, Mohit Iyyer",
  "next_check_at": "2026-11-16T00:00:00.000Z",
  "known_limitations": [
    "Abstract page only; the annotation release location and license were not found.",
    "About books, not screenplays or TV canon."
  ],
  "upstream_updated_at": "2024-09-30T17:39:59.000Z"
}