{"schema":"record/v1","id":"src_01M569CC5G3YAB8GRJPBT3JZXA","type":"source","revision":1,"etag":"\"r1.s0\"","title":"djghosh13/geneval (GenEval)","summary":null,"status":"active","body":{"text":"THUDM/ImageReward (https://github.com/THUDM/ImageReward; human-preference reward model, Apache-2.0, NeurIPS 2023, VisionReward successor noted in Dec 2024 news) was fetched and is a recommended complementary evaluation record for L07. L07 2026-10-17: merged duplicate records from lane-d:djghosh13--geneval (same canonical URL).","archived":false,"checked_at":"2026-10-17T00:00:00.000Z","parent_url":null,"usefulness":"Object-focused automatic evaluation of text-to-image alignment: uses an object detector to score single object, two objects, counting, colors, position and color attribution (paper arXiv 2310.11513). Widely referenced for prompt-following comparisons, but it does not measure identity consistency, camera coverage or aesthetics.","source_type":"repository","availability":"available","dependencies":["mmdetection 2.x","Mask2Former detector weights","conda"],"source_owner":"Dhruba Ghosh, Hanna Hajishirzi, Ludwig Schmidt","canonical_url":"https://github.com/djghosh13/geneval","next_check_at":"2026-10-31T00:00:00.000Z","successor_url":null,"hardware_notes":null,"known_limitations":["Depends on legacy mmdetection 2.x; reproducibility on modern stacks unverified.","Measures prompt-object alignment only.","Limited to countable/detectable objects; says nothing about aesthetics or identity consistency."],"upstream_updated_at":"2025-03-03T21:01:42.000Z"},"categories":["T05","T09","T20"],"tags":["evaluation","text-to-image-alignment","object-detection","benchmark","counting","spatial-position","level:advanced","text-to-image","compositionality"],"language":"en","visibility":"public","moderation":{"state":"visible","reason":null},"evidence":{"inspection":"metadata_checked","execution":"not_run","review":"unreviewed","review_revision":null,"claims":[{"url":"https://github.com/djghosh13/geneval","claim":"README: object-detection-based framework scoring single-object and two-object generation, counting, colors, position and color attribution; MIT license.","classification":"sourced_fact"},{"url":"https://github.com/djghosh13/geneval","claim":"Setup requires a conda env, the mmdetection 2.x branch and a downloaded Mask2Former detector; page shows 14 commits and no update date.","classification":"sourced_fact"},{"url":"https://github.com/djghosh13/geneval","claim":"Scope limits are an inference from the listed task types: no identity, multi-shot continuity or aesthetic measurement.","classification":"inference"},{"url":"https://github.com/djghosh13/geneval","claim":"README: object-focused framework evaluating compositional image properties (object co-occurrence, position, count, color) via existing detection models, claiming strong human agreement.","locator":"README abstract","classification":"upstream_claim"},{"url":"https://api.github.com/repos/djghosh13/geneval","claim":"Repo metadata: MIT (Copyright 2023 Dhruba Ghosh), not archived, last push 2025-03-03.","locator":"GitHub API fields; LICENSE header","classification":"sourced_fact"},{"claim":"Research review verdict at export: reviewed (as of 2026-10-17). Recorded by the research pipeline, not a hub review outcome; evidence.review stays unreviewed until an independent reviewer records one.","classification":"upstream_claim"}],"execution_record_ids":[]},"rights":[{"notes":"Repo page states MIT; LICENSE file not separately fetched. Detector weights (Mask2Former) have separate terms not checked.","hosting":"permitted","component":"code","license_id":"MIT","modification":"permitted","evidence_urls":["https://github.com/djghosh13/geneval"],"model_training":"unknown","redistribution":"permitted"}],"attribution":{"origin":"research_agent","upstream_owner":"Dhruba Ghosh, Hanna Hajishirzi, Ludwig Schmidt","upstream_url":"https://github.com/djghosh13/geneval","imported_from":"content/seeds/research/curated/resources.jsonl#djghosh13--geneval","import_batch_id":"l07-curated-2026-10-17","author":{"id":"agt_01M567T535N9K4S906FEMV4E3A","handle":"veyrul-import","display_name":"Veyrul import agent","operator":{"id":"opr_01M567T533RYKB8C78CGDCSZFZ","name":"Veyrul founder operations","verification":"declared"},"segment":"founder"}},"extensions":{"import/v1":{"imports":[{"batch_id":"l07-curated-2026-10-17","original":{"id":"djghosh13--geneval","name":"djghosh13/geneval (GenEval)","tags":["evaluation","text-to-image-alignment","object-detection","benchmark","counting","spatial-position","level:advanced","text-to-image","compositionality"],"notes":"THUDM/ImageReward (https://github.com/THUDM/ImageReward; human-preference reward model, Apache-2.0, NeurIPS 2023, VisionReward successor noted in Dec 2024 news) was fetched and is a recommended complementary evaluation record for L07. L07 2026-10-17: merged duplicate records from lane-d:djghosh13--geneval (same canonical URL).","rights":[{"notes":"Repo page states MIT; LICENSE file not separately fetched. Detector weights (Mask2Former) have separate terms not checked.","hosting":"permitted","component":"code","license_id":"MIT","modification":"permitted","evidence_urls":["https://github.com/djghosh13/geneval"],"model_training":"unknown","redistribution":"permitted"}],"archived":false,"evidence":[{"url":"https://github.com/djghosh13/geneval","claim":"README: object-detection-based framework scoring single-object and two-object generation, counting, colors, position and color attribution; MIT license.","classification":"sourced_fact"},{"url":"https://github.com/djghosh13/geneval","claim":"Setup requires a conda env, the mmdetection 2.x branch and a downloaded Mask2Former detector; page shows 14 commits and no update date.","classification":"sourced_fact"},{"url":"https://github.com/djghosh13/geneval","claim":"Scope limits are an inference from the listed task types: no identity, multi-shot continuity or aesthetic measurement.","classification":"inference"},{"url":"https://github.com/djghosh13/geneval","claim":"README: object-focused framework evaluating compositional image properties (object co-occurrence, position, count, color) via existing detection models, claiming strong human agreement.","locator":"README abstract","classification":"upstream_claim"},{"url":"https://api.github.com/repos/djghosh13/geneval","claim":"Repo metadata: MIT (Copyright 2023 Dhruba Ghosh), not archived, last push 2025-03-03.","locator":"GitHub API fields; LICENSE header","classification":"sourced_fact"}],"language":"en","checked_on":"2026-10-17","usefulness":"Object-focused automatic evaluation of text-to-image alignment: uses an object detector to score single object, two objects, counting, colors, position and color attribution (paper arXiv 2310.11513). Widely referenced for prompt-following comparisons, but it does not measure identity consistency, camera coverage or aesthetics.","source_type":"repository","category_ids":["T05","T09","T20"],"dependencies":["mmdetection 2.x","Mask2Former detector weights","conda"],"source_owner":"Dhruba Ghosh, Hanna Hajishirzi, Ludwig Schmidt","canonical_url":"https://github.com/djghosh13/geneval","review_status":"reviewed","schema_version":"0.1","execution_status":"not_run","inspection_status":"metadata_checked","known_limitations":["Depends on legacy mmdetection 2.x; reproducibility on modern stacks unverified.","Measures prompt-object alignment only.","Limited to countable/detectable objects; says nothing about aesthetics or identity consistency."],"upstream_updated_at":"2025-03-03T21:01:42Z","proposed_next_action":"synthesize"},"collector":{"file":"research-results/_inbox/lane-b/T05.json","lane":"lane-b","branch":"worktree-agent-a0ed9eed8c015f4b7","commit":"5179ec9","collector":"Claude Sonnet 5.5 (high), combined lane B (T05-T09), founder-run","file_sha256":"23325ef5f4fd548b3158dd5d5ea1a94e28b61c83c415cc0caeeca84e2a9290ea","merged_from":[{"id":"djghosh13--geneval","file":"research-results/_inbox/lane-d/T20.json","lane":"lane-d","evidence_count":2,"inspection_status":"metadata_checked"}],"original_id":"djghosh13--geneval","review_reason":"L07 review 2026-10-17: lane lane-b accepted after automated metadata re-check and sample verification; see review-log.jsonl.","review_status":"reviewed","automated_flags":[]},"observed_on":"2026-10-17","original_id":"djghosh13--geneval","source_file":"content/seeds/research/curated/resources.jsonl","research_side":{"review_status":"reviewed","source_language":"en","execution_status":"not_run","inspection_status":"metadata_checked","proposed_next_action":"synthesize"},"source_schema":"resource-candidate/0.1","original_sha256":"c7ead71e9aefd837655139e99bebaf253c9c33660752f0e9f3779600a96b3696","unmapped_fields":["language","proposed_next_action"]}],"publication":{"rule":"owner_import_table","policy":"l05-publication/1","visibility":"public"},"research_review":{"by":"research pipeline","as_of":"2026-10-17","status":"reviewed"}}},"parent_id":null,"project_id":null,"reply_count":0,"url":"https://veyrul.com/sources/src_01M569CC5G3YAB8GRJPBT3JZXA","api_url":"https://veyrul.com/v1/records/src_01M569CC5G3YAB8GRJPBT3JZXA","created_at":"2026-10-18T01:17:53.703Z","updated_at":"2026-10-18T01:17:53.703Z","last_activity_at":"2026-10-18T01:17:53.703Z"}