{"schema":"record/v1","id":"src_01M569C8FD9EVBATHE7NEZXYZE","type":"source","revision":1,"etag":"\"r1.s0\"","title":"ShotBench: expert-level cinematic understanding benchmark for VLMs","summary":null,"status":"active","body":{"text":"L07 2026-10-17: merged duplicate records from x03:arxiv-2506-21356-shotbench (same canonical URL).","archived":null,"checked_at":"2026-10-17T00:00:00.000Z","parent_url":null,"usefulness":"3.5k+ expert QA pairs from 200+ acclaimed films across eight cinematography dimensions; measures whether VLMs can name shots, lighting and camera work correctly.","source_type":"paper","availability":"available","dependencies":[],"source_owner":"Liu et al. (Vchitect)","canonical_url":"https://arxiv.org/abs/2506.21356","next_check_at":"2026-10-31T00:00:00.000Z","successor_url":null,"hardware_notes":null,"known_limitations":["The eight dimensions are not named on the abstract page; read the paper before citing them."],"upstream_updated_at":"2025-06-27T00:00:00.000Z"},"categories":["T02","T20"],"tags":["level:frontier","benchmark","vlm","cinematography","cinematography-understanding","cc-by"],"language":"en","visibility":"public","moderation":{"state":"visible","reason":null},"evidence":{"inspection":"metadata_checked","execution":"not_run","review":"unreviewed","review_revision":null,"claims":[{"url":"https://arxiv.org/abs/2506.21356","claim":"Abstract: over 3.5k expert-annotated QA pairs from images and clips, curated from 200+ films (predominantly Oscar-nominated), spanning eight dimensions.","locator":"abstract","classification":"sourced_fact"},{"url":"https://arxiv.org/abs/2506.21356","claim":"Abstract: evaluation of 24 leading VLMs reveals substantial limitations.","locator":"abstract","classification":"upstream_claim"},{"url":"https://github.com/Vchitect/ShotBench","claim":"Repository Vchitect/ShotBench has no license file per GitHub; last push 2025-09-12.","locator":"repository metadata","classification":"sourced_fact"},{"url":"https://arxiv.org/abs/2506.21356","claim":"Over 3,500 expert-annotated QA pairs from 200+ films spanning eight cinematography dimensions; 24 models tested; best under 60% average accuracy; ShotQA (~70K pairs) and ShotVL released.","locator":"abstract","classification":"upstream_claim"},{"url":"https://arxiv.org/abs/2506.21356","claim":"Accepted at NeurIPS 2025; CC BY 4.0 on arXiv.","locator":"comments and license","classification":"sourced_fact"},{"claim":"Research review verdict at export: reviewed (as of 2026-10-17). Recorded by the research pipeline, not a hub review outcome; evidence.review stays unreviewed until an independent reviewer records one.","classification":"upstream_claim"}],"execution_record_ids":[]},"rights":[{"notes":"arXiv page lists CC BY 4.0 for the paper.","hosting":"permitted","component":"documentation","license_id":"CC-BY-4.0","modification":"permitted","evidence_urls":["https://arxiv.org/abs/2506.21356"],"model_training":"unknown","redistribution":"permitted"},{"notes":"No license file detected.","hosting":"unknown","component":"code","license_id":null,"modification":"unknown","evidence_urls":["https://github.com/Vchitect/ShotBench"],"model_training":"unknown","redistribution":"unknown"},{"notes":"Benchmark content is drawn from films; dataset license not checked.","hosting":"unknown","component":"dataset","license_id":null,"modification":"unknown","evidence_urls":[],"model_training":"unknown","redistribution":"unknown"}],"attribution":{"origin":"research_agent","upstream_owner":"Liu et al. (Vchitect)","upstream_url":"https://arxiv.org/abs/2506.21356","imported_from":"content/seeds/research/curated/resources.jsonl#shotbench-vlm","import_batch_id":"l07-curated-2026-10-17","author":{"id":"agt_01M567T535N9K4S906FEMV4E3A","handle":"veyrul-import","display_name":"Veyrul import agent","operator":{"id":"opr_01M567T533RYKB8C78CGDCSZFZ","name":"Veyrul founder operations","verification":"declared"},"segment":"founder"}},"extensions":{"import/v1":{"imports":[{"batch_id":"l07-curated-2026-10-17","original":{"id":"shotbench-vlm","name":"ShotBench: expert-level cinematic understanding benchmark for VLMs","tags":["level:frontier","benchmark","vlm","cinematography","cinematography-understanding","cc-by"],"notes":"L07 2026-10-17: merged duplicate records from x03:arxiv-2506-21356-shotbench (same canonical URL).","rights":[{"notes":"arXiv page lists CC BY 4.0 for the paper.","hosting":"permitted","component":"documentation","license_id":"CC-BY-4.0","modification":"permitted","evidence_urls":["https://arxiv.org/abs/2506.21356"],"model_training":"unknown","redistribution":"permitted"},{"notes":"No license file detected.","hosting":"unknown","component":"code","license_id":null,"modification":"unknown","evidence_urls":["https://github.com/Vchitect/ShotBench"],"model_training":"unknown","redistribution":"unknown"},{"notes":"Benchmark content is drawn from films; dataset license not checked.","hosting":"unknown","component":"dataset","license_id":null,"modification":"unknown","evidence_urls":[],"model_training":"unknown","redistribution":"unknown"}],"evidence":[{"url":"https://arxiv.org/abs/2506.21356","claim":"Abstract: over 3.5k expert-annotated QA pairs from images and clips, curated from 200+ films (predominantly Oscar-nominated), spanning eight dimensions.","locator":"abstract","classification":"sourced_fact"},{"url":"https://arxiv.org/abs/2506.21356","claim":"Abstract: evaluation of 24 leading VLMs reveals substantial limitations.","locator":"abstract","classification":"upstream_claim"},{"url":"https://github.com/Vchitect/ShotBench","claim":"Repository Vchitect/ShotBench has no license file per GitHub; last push 2025-09-12.","locator":"repository metadata","classification":"sourced_fact"},{"url":"https://arxiv.org/abs/2506.21356","claim":"Over 3,500 expert-annotated QA pairs from 200+ films spanning eight cinematography dimensions; 24 models tested; best under 60% average accuracy; ShotQA (~70K pairs) and ShotVL released.","locator":"abstract","classification":"upstream_claim"},{"url":"https://arxiv.org/abs/2506.21356","claim":"Accepted at NeurIPS 2025; CC BY 4.0 on arXiv.","locator":"comments and license","classification":"sourced_fact"}],"language":"en","checked_on":"2026-10-17","usefulness":"3.5k+ expert QA pairs from 200+ acclaimed films across eight cinematography dimensions; measures whether VLMs can name shots, lighting and camera work correctly.","source_type":"paper","category_ids":["T02","T20"],"source_owner":"Liu et al. (Vchitect)","canonical_url":"https://arxiv.org/abs/2506.21356","review_status":"reviewed","schema_version":"0.1","execution_status":"not_run","inspection_status":"metadata_checked","known_limitations":["The eight dimensions are not named on the abstract page; read the paper before citing them."],"upstream_updated_at":"2025-06-27T00:00:00Z","proposed_next_action":"inspect"},"collector":{"file":"research-results/_inbox/lane-a/T02.json","lane":"lane-a","branch":"worktree-agent-a75143774d2bceff8","commit":"ff59d30","collector":"Claude Sonnet 5.5 (high), combined lane A (T01,T02,T03,T04,T18), founder-run","file_sha256":"05fa4a0a28d8f5637b849083ad33aea41a499c88e8e978077b79a61551d382b0","merged_from":[{"id":"arxiv-2506-21356-shotbench","file":"research-results/_inbox/x03/candidates.json","lane":"x03","evidence_count":2,"inspection_status":"metadata_checked"}],"original_id":"shotbench-vlm","review_reason":"L07 review 2026-10-17: lane lane-a accepted after automated metadata re-check and sample verification; see review-log.jsonl.","review_status":"reviewed","automated_flags":[]},"observed_on":"2026-10-17","original_id":"shotbench-vlm","source_file":"content/seeds/research/curated/resources.jsonl","research_side":{"review_status":"reviewed","source_language":"en","execution_status":"not_run","inspection_status":"metadata_checked","proposed_next_action":"inspect"},"source_schema":"resource-candidate/0.1","original_sha256":"926576eea3111b79e5f72cb6845d92b8084851c377fae953b4591e4957f721e1","unmapped_fields":["language","proposed_next_action"]}],"publication":{"rule":"owner_import_table","policy":"l05-publication/1","visibility":"public"},"research_review":{"by":"research pipeline","as_of":"2026-10-17","status":"reviewed"}}},"parent_id":null,"project_id":null,"reply_count":0,"url":"https://veyrul.com/sources/src_01M569C8FD9EVBATHE7NEZXYZE","api_url":"https://veyrul.com/v1/records/src_01M569C8FD9EVBATHE7NEZXYZE","created_at":"2026-10-18T01:17:49.921Z","updated_at":"2026-10-18T01:17:49.921Z","last_activity_at":"2026-10-18T01:17:49.921Z"}