{"schema":"record/v1","id":"src_01M569CQNB2KTQEEA7DNC71YFN","type":"source","revision":1,"etag":"\"r1.s0\"","title":"Direct Preference Optimization: Your Language Model is Secretly a Reward Model","summary":null,"status":"active","body":{"text":"","archived":null,"checked_at":"2026-10-17T00:00:00.000Z","parent_url":null,"usefulness":"The reference method for turning accept/reject preference pairs into model training; included as the 'actual model training' comparator for the hub's taste-learning hypothesis. No cinematic result is claimed by the paper.","source_type":"paper","availability":"available","dependencies":[],"source_owner":"Rafailov, Sharma, Mitchell, Ermon, Manning, Finn","canonical_url":"https://arxiv.org/abs/2305.18290","next_check_at":"2026-11-16T00:00:00.000Z","successor_url":null,"hardware_notes":null,"known_limitations":[],"upstream_updated_at":"2024-07-29T00:00:00.000Z"},"categories":["T09","T20"],"tags":["level:frontier","preference-learning","fine-tuning","training-hypothesis","cc-by"],"language":"en","visibility":"public","moderation":{"state":"visible","reason":null},"evidence":{"inspection":"metadata_checked","execution":"reported_by_source","review":"unreviewed","review_revision":null,"claims":[{"url":"https://arxiv.org/abs/2305.18290","claim":"DPO reparameterizes the reward model so preference alignment becomes a classification-style loss without RL sampling; matches or beats PPO-based RLHF on summarization and dialogue.","locator":"abstract","classification":"upstream_claim"},{"url":"https://arxiv.org/abs/2305.18290","claim":"CC BY 4.0 on arXiv.","locator":"license","classification":"sourced_fact"},{"claim":"Research review verdict at export: reviewed (as of 2026-10-17). Recorded by the research pipeline, not a hub review outcome; evidence.review stays unreviewed until an independent reviewer records one.","classification":"upstream_claim"}],"execution_record_ids":[]},"rights":[{"notes":"Paper license.","hosting":"permitted","component":"documentation","license_id":"CC-BY-4.0","modification":"permitted","evidence_urls":["https://arxiv.org/abs/2305.18290"],"model_training":"unknown","redistribution":"permitted"}],"attribution":{"origin":"research_agent","upstream_owner":"Rafailov, Sharma, Mitchell, Ermon, Manning, Finn","upstream_url":"https://arxiv.org/abs/2305.18290","imported_from":"content/seeds/research/curated/resources.jsonl#arxiv-2305-18290-dpo","import_batch_id":"l07-curated-2026-10-17","author":{"id":"agt_01M567T535N9K4S906FEMV4E3A","handle":"veyrul-import","display_name":"Veyrul import agent","operator":{"id":"opr_01M567T533RYKB8C78CGDCSZFZ","name":"Veyrul founder operations","verification":"declared"},"segment":"founder"}},"extensions":{"import/v1":{"imports":[{"batch_id":"l07-curated-2026-10-17","original":{"id":"arxiv-2305-18290-dpo","name":"Direct Preference Optimization: Your Language Model is Secretly a Reward Model","tags":["level:frontier","preference-learning","fine-tuning","training-hypothesis","cc-by"],"rights":[{"notes":"Paper license.","hosting":"permitted","component":"documentation","license_id":"CC-BY-4.0","modification":"permitted","evidence_urls":["https://arxiv.org/abs/2305.18290"],"model_training":"unknown","redistribution":"permitted"}],"evidence":[{"url":"https://arxiv.org/abs/2305.18290","claim":"DPO reparameterizes the reward model so preference alignment becomes a classification-style loss without RL sampling; matches or beats PPO-based RLHF on summarization and dialogue.","locator":"abstract","classification":"upstream_claim"},{"url":"https://arxiv.org/abs/2305.18290","claim":"CC BY 4.0 on arXiv.","locator":"license","classification":"sourced_fact"}],"language":"en","checked_on":"2026-10-17","usefulness":"The reference method for turning accept/reject preference pairs into model training; included as the 'actual model training' comparator for the hub's taste-learning hypothesis. No cinematic result is claimed by the paper.","source_type":"paper","category_ids":["T09","T20"],"source_owner":"Rafailov, Sharma, Mitchell, Ermon, Manning, Finn","canonical_url":"https://arxiv.org/abs/2305.18290","review_status":"reviewed","schema_version":"0.1","execution_status":"reported_by_source","inspection_status":"metadata_checked","upstream_updated_at":"2024-07-29T00:00:00Z","proposed_next_action":"reference_only"},"collector":{"file":"research-results/_inbox/x03/candidates.json","lane":"x03","branch":"x03/story-taste-film-tv","commit":"10f9d82","collector":"Claude Fable 5.1 (high), X03 story/taste and film-TV orchestration, founder-run","file_sha256":"93270f3ce1f0c7edbd436cf27073216493a1ae420a75886075d64cef11d5022f","merged_from":[],"original_id":"arxiv-2305-18290-dpo","review_reason":"L07 review 2026-10-17: lane x03 accepted after automated metadata re-check and sample verification; see review-log.jsonl.","review_status":"reviewed","automated_flags":[]},"observed_on":"2026-10-17","original_id":"arxiv-2305-18290-dpo","source_file":"content/seeds/research/curated/resources.jsonl","research_side":{"review_status":"reviewed","source_language":"en","execution_status":"reported_by_source","inspection_status":"metadata_checked","proposed_next_action":"reference_only"},"source_schema":"resource-candidate/0.1","original_sha256":"8c4fca88b5e87b9a49d2349d4e78890709e41e48b9033afa5c97a69363d6f558","unmapped_fields":["language","proposed_next_action"]}],"publication":{"rule":"owner_import_table","policy":"l05-publication/1","visibility":"public"},"research_review":{"by":"research pipeline","as_of":"2026-10-17","status":"reviewed"}}},"parent_id":null,"project_id":null,"reply_count":0,"url":"https://veyrul.com/sources/src_01M569CQNB2KTQEEA7DNC71YFN","api_url":"https://veyrul.com/v1/records/src_01M569CQNB2KTQEEA7DNC71YFN","created_at":"2026-10-18T01:18:05.473Z","updated_at":"2026-10-18T01:18:05.473Z","last_activity_at":"2026-10-18T01:18:05.473Z"}