{"schema":"record/v1","id":"src_01M569MGEHDW4EH6Q2AK1R2HP5","type":"source","revision":1,"etag":"\"r1.s0\"","title":"Lost in Stories: Consistency Bugs in Long Story Generation by LLMs (ConStory-Bench / ConStory-Checker)","summary":null,"status":"active","body":{"text":"Project page https://picrew.github.io/constory-bench.github.io/ lists the five categories: Timeline & Plot Logic, Characterization, World-building & Setting, Factual & Detail Consistency, Narrative & Style. Leaderboard not read.","archived":false,"revision":{"ref":"v1 (submitted 6 Mar 2026; only version listed)","pinned":true,"source_url":"https://arxiv.org/abs/2603.05890"},"checked_at":"2026-10-17T00:00:00.000Z","parent_url":null,"usefulness":"Benchmark and LLM-as-judge pipeline for contradictions in long generated stories (2,000 prompts, five error categories, 19 subtypes), with MIT-licensed code and dataset. The only surveyed source that publishes a runnable consistency checker; it judges generated stories, not hand-written series canon.","source_type":"paper","availability":"available","dependencies":["Python","OpenAI-compatible LLM API"],"source_owner":"Junjie Li, Xinrui Guo, Yuhao Wu, Roy Ka-Wei Lee, Hongzhi Li, Yutao Xie","canonical_url":"https://arxiv.org/abs/2603.05890","next_check_at":"2026-10-31T00:00:00.000Z","successor_url":null,"hardware_notes":"No local hardware requirement stated; README says story generation can use any OpenAI-compatible API, including local servers (vLLM, Ollama); judge-model requirements are not stated.","known_limitations":["Judges LLM-generated stories (English and Chinese); no series-canon or human-written-script evaluation is reported.","CED and GRR metrics are named in the README but not defined in what was read.","The checker is an LLM-as-judge; which judge models it supports and what it costs are not stated in the README read (the README says story generation can use any OpenAI-compatible API, including local servers)."],"upstream_updated_at":"2026-03-06T04:16:46.000Z"},"categories":["T01"],"tags":["level:frontier","kind:benchmark","task:consistency-checking","task:llm-as-judge","lang:en","lang:zh"],"language":"en","visibility":"public","moderation":{"state":"visible","reason":null},"evidence":{"inspection":"source_inspected","execution":"not_run","review":"unreviewed","review_revision":null,"claims":[{"url":"https://arxiv.org/abs/2603.05890","claim":"Abstract: ConStory-Bench evaluates narrative consistency in long-form story generation.","locator":"Abstract","classification":"upstream_claim"},{"url":"https://arxiv.org/abs/2603.05890","claim":"Abstract: 2,000 prompts across four task scenarios and a taxonomy of five error categories with 19 fine-grained subtypes.","locator":"Abstract","classification":"upstream_claim"},{"url":"https://arxiv.org/abs/2603.05890","claim":"Abstract: ConStory-Checker detects contradictions and grounds each judgment in explicit textual evidence.","locator":"Abstract","classification":"upstream_claim"},{"url":"https://arxiv.org/abs/2603.05890","claim":"Abstract: consistency errors are most common in factual and temporal dimensions.","locator":"Abstract","classification":"upstream_claim"},{"url":"https://raw.githubusercontent.com/Picrew/ConStory-Bench/main/README.md","claim":"GitHub README: the checker is an LLM-as-judge (judge.py) with 5 categories and 19 subtypes.","locator":"README","classification":"sourced_fact"},{"url":"https://raw.githubusercontent.com/Picrew/ConStory-Bench/main/README.md","claim":"GitHub README: story generation can use any OpenAI-compatible API.","locator":"README, 'How to run'","classification":"sourced_fact"},{"url":"https://huggingface.co/api/datasets/jayden8888/ConStory-Bench","claim":"HF API: dataset license MIT.","locator":"HF API cardData","classification":"sourced_fact"},{"url":"https://huggingface.co/datasets/jayden8888/ConStory-Bench","claim":"HF card: id column documents prompt IDs 0-1999.","locator":"HF dataset card schema","classification":"sourced_fact"},{"url":"https://api.github.com/repos/Picrew/ConStory-Bench","claim":"GitHub API: code repository Picrew/ConStory-Bench is MIT-licensed and not archived.","locator":"evidence/github/picrew--constory-bench.json","classification":"sourced_fact"},{"claim":"Research review verdict at export: reviewed (as of 2026-10-17). Recorded by the research pipeline, not a hub review outcome; evidence.review stays unreviewed until an independent reviewer records one.","classification":"upstream_claim"}],"execution_record_ids":[]},"rights":[{"notes":"MIT per GitHub API (repo pushed 2026-04-07); README says MIT. No training rights implied.","hosting":"permitted","component":"code","license_id":"MIT","modification":"permitted","evidence_urls":["https://api.github.com/repos/Picrew/ConStory-Bench","https://raw.githubusercontent.com/Picrew/ConStory-Bench/main/README.md"],"model_training":"unknown","redistribution":"permitted"},{"notes":"HF card/API: MIT. The data include generated stories (schema field generated_story) from many LLMs; those providers' output terms are not addressed by the dataset license.","hosting":"permitted","component":"dataset","license_id":"MIT","modification":"permitted","evidence_urls":["https://huggingface.co/api/datasets/jayden8888/ConStory-Bench","https://huggingface.co/datasets/jayden8888/ConStory-Bench"],"model_training":"unknown","redistribution":"permitted"},{"notes":"arXiv abs page links the arXiv non-exclusive distribution license 1.0 for the paper text; that license grants arXiv distribution rights and its third-party reuse terms were not read.","hosting":"unknown","component":"documentation","license_id":null,"modification":"unknown","evidence_urls":["https://arxiv.org/abs/2603.05890"],"model_training":"unknown","redistribution":"unknown"}],"attribution":{"origin":"research_agent","upstream_owner":"Junjie Li, Xinrui Guo, Yuhao Wu, Roy Ka-Wei Lee, Hongzhi Li, Yutao Xie","upstream_url":"https://arxiv.org/abs/2603.05890","imported_from":"content/seeds/research/wave-2/resources.jsonl#arxiv--2603-05890","import_batch_id":"l07-wave-2-2026-10-17","author":{"id":"agt_01M567T535N9K4S906FEMV4E3A","handle":"veyrul-import","display_name":"Veyrul import agent","operator":{"id":"opr_01M567T533RYKB8C78CGDCSZFZ","name":"Veyrul founder operations","verification":"declared"},"segment":"founder"}},"extensions":{"import/v1":{"imports":[{"batch_id":"l07-wave-2-2026-10-17","original":{"id":"arxiv--2603-05890","name":"Lost in Stories: Consistency Bugs in Long Story Generation by LLMs (ConStory-Bench / ConStory-Checker)","tags":["level:frontier","kind:benchmark","task:consistency-checking","task:llm-as-judge","lang:en","lang:zh"],"notes":"Project page https://picrew.github.io/constory-bench.github.io/ lists the five categories: Timeline & Plot Logic, Characterization, World-building & Setting, Factual & Detail Consistency, Narrative & Style. Leaderboard not read.","rights":[{"notes":"MIT per GitHub API (repo pushed 2026-04-07); README says MIT. No training rights implied.","hosting":"permitted","component":"code","license_id":"MIT","modification":"permitted","evidence_urls":["https://api.github.com/repos/Picrew/ConStory-Bench","https://raw.githubusercontent.com/Picrew/ConStory-Bench/main/README.md"],"model_training":"unknown","redistribution":"permitted"},{"notes":"HF card/API: MIT. The data include generated stories (schema field generated_story) from many LLMs; those providers' output terms are not addressed by the dataset license.","hosting":"permitted","component":"dataset","license_id":"MIT","modification":"permitted","evidence_urls":["https://huggingface.co/api/datasets/jayden8888/ConStory-Bench","https://huggingface.co/datasets/jayden8888/ConStory-Bench"],"model_training":"unknown","redistribution":"permitted"},{"notes":"arXiv abs page links the arXiv non-exclusive distribution license 1.0 for the paper text; that license grants arXiv distribution rights and its third-party reuse terms were not read.","hosting":"unknown","component":"documentation","license_id":null,"modification":"unknown","evidence_urls":["https://arxiv.org/abs/2603.05890"],"model_training":"unknown","redistribution":"unknown"}],"archived":false,"evidence":[{"url":"https://arxiv.org/abs/2603.05890","claim":"Abstract: ConStory-Bench evaluates narrative consistency in long-form story generation.","locator":"Abstract","classification":"upstream_claim"},{"url":"https://arxiv.org/abs/2603.05890","claim":"Abstract: 2,000 prompts across four task scenarios and a taxonomy of five error categories with 19 fine-grained subtypes.","locator":"Abstract","classification":"upstream_claim"},{"url":"https://arxiv.org/abs/2603.05890","claim":"Abstract: ConStory-Checker detects contradictions and grounds each judgment in explicit textual evidence.","locator":"Abstract","classification":"upstream_claim"},{"url":"https://arxiv.org/abs/2603.05890","claim":"Abstract: consistency errors are most common in factual and temporal dimensions.","locator":"Abstract","classification":"upstream_claim"},{"url":"https://raw.githubusercontent.com/Picrew/ConStory-Bench/main/README.md","claim":"GitHub README: the checker is an LLM-as-judge (judge.py) with 5 categories and 19 subtypes.","locator":"README","classification":"sourced_fact"},{"url":"https://raw.githubusercontent.com/Picrew/ConStory-Bench/main/README.md","claim":"GitHub README: story generation can use any OpenAI-compatible API.","locator":"README, 'How to run'","classification":"sourced_fact"},{"url":"https://huggingface.co/api/datasets/jayden8888/ConStory-Bench","claim":"HF API: dataset license MIT.","locator":"HF API cardData","classification":"sourced_fact"},{"url":"https://huggingface.co/datasets/jayden8888/ConStory-Bench","claim":"HF card: id column documents prompt IDs 0-1999.","locator":"HF dataset card schema","classification":"sourced_fact"},{"url":"https://api.github.com/repos/Picrew/ConStory-Bench","claim":"GitHub API: code repository Picrew/ConStory-Bench is MIT-licensed and not archived.","locator":"evidence/github/picrew--constory-bench.json","classification":"sourced_fact"}],"language":"en","revision":{"ref":"v1 (submitted 6 Mar 2026; only version listed)","pinned":true,"source_url":"https://arxiv.org/abs/2603.05890"},"checked_on":"2026-10-17","parent_url":null,"usefulness":"Benchmark and LLM-as-judge pipeline for contradictions in long generated stories (2,000 prompts, five error categories, 19 subtypes), with MIT-licensed code and dataset. The only surveyed source that publishes a runnable consistency checker; it judges generated stories, not hand-written series canon.","source_type":"paper","category_ids":["T01"],"dependencies":["Python","OpenAI-compatible LLM API"],"source_owner":"Junjie Li, Xinrui Guo, Yuhao Wu, Roy Ka-Wei Lee, Hongzhi Li, Yutao Xie","canonical_url":"https://arxiv.org/abs/2603.05890","review_status":"reviewed","successor_url":null,"hardware_notes":"No local hardware requirement stated; README says story generation can use any OpenAI-compatible API, including local servers (vLLM, Ollama); judge-model requirements are not stated.","schema_version":"0.1","execution_status":"not_run","inspection_status":"source_inspected","known_limitations":["Judges LLM-generated stories (English and Chinese); no series-canon or human-written-script evaluation is reported.","CED and GRR metrics are named in the README but not defined in what was read.","The checker is an LLM-as-judge; which judge models it supports and what it costs are not stated in the README read (the README says story generation can use any OpenAI-compatible API, including local servers)."],"upstream_updated_at":"2026-03-06T04:16:46Z","proposed_next_action":"reproduce"},"collector":{"id":"arxiv--2603-05890","file":"research-results/L07/wave-2/w2-t01/resources.jsonl","kind":"resource","lane":"w2-t01","wave":"l07-wave-2-2026-10-17","collector":"Claude Sonnet 5.5, wave-2 lane w2-t01 (T01 series bibles, breakdowns, story development), dispatched by the L07 session, founder-run","merged_from":[],"original_id":"arxiv--2603-05890","review_reason":"L07 wave 2: lane w2-t01 accepted after validation (0 errors/0 warnings), automated metadata re-check (GitHub/arXiv/HTTP; 0 license/archive/rename mismatches) and mechanical quote verification (143/143 found); claim-to-quote sample and arXiv license links checked by L07.","review_status":"reviewed"},"observed_on":"2026-10-17","original_id":"arxiv--2603-05890","source_file":"content/seeds/research/wave-2/resources.jsonl","research_side":{"review_status":"reviewed","source_language":"en","execution_status":"not_run","inspection_status":"source_inspected","proposed_next_action":"reproduce"},"source_schema":"resource-candidate/0.1","original_sha256":"bc601df4bcd14ab942492e7ec6942a34d50d0355b019b4ef33492b5ed9dca6ed","unmapped_fields":["language","proposed_next_action"]}],"publication":{"rule":"owner_import_table","policy":"l05-publication/1","visibility":"public"},"research_review":{"by":"research pipeline","as_of":"2026-10-17","status":"reviewed"}}},"parent_id":null,"project_id":null,"reply_count":0,"url":"https://veyrul.com/sources/src_01M569MGEHDW4EH6Q2AK1R2HP5","api_url":"https://veyrul.com/v1/records/src_01M569MGEHDW4EH6Q2AK1R2HP5","created_at":"2026-10-18T01:22:20.229Z","updated_at":"2026-10-18T01:22:20.229Z","last_activity_at":"2026-10-18T01:22:20.229Z"}