https://www.hansking.cn/blog A posts created with Next.js and Tailwind.css en-us admin@hansking.cn (HANS) admin@hansking.cn (HANS) Sun, 04 Aug 2024 10:00:00 GMT https://www.hansking.cn/blog/2026pian-hao-you-hua-cong-RLHF-dao-DPO https://www.hansking.cn/blog/2026pian-hao-you-hua-cong-RLHF-dao-DPO [{"value":"偏好优化：从 RLHF 到 DPO","url":"#偏好优化从-rlhf-到-dpo","depth":1},{"value":"一、预训练和 SFT 解决了什么","url":"#一预训练和-sft-解决了什么","depth":2},{"value":"二、偏好数据把绝对打分改成相对比较","url":"#二偏好数据把绝对打分改成相对比较","depth":2},{"value":"三、奖励模型：把偏好变成一个分数","url":"#三奖励模型把偏好变成一个分数","depth":2},{"value":"四、PPO 为什么被用于 RLHF","url":"#四ppo-为什么被用于-rlhf","depth":2},{"value":"五、DPO 的直觉：直接让模型偏爱 chosen","url":"#五dpo-的直觉直接让模型偏爱-chosen","depth":2},{"value":"六、三种路线放在一起看","url":"#六三种路线放在一起看","depth":2},{"value":"七、为什么数据质量比算法名字更重要","url":"#七为什么数据质量比算法名字更重要","depth":2},{"value":"偏好差异不清楚","url":"#偏好差异不清楚","depth":3},{"value":"chosen 只是更长","url":"#chosen-只是更长","depth":3},{"value":"rejected 过于糟糕","url":"#rejected-过于糟糕","depth":3},{"value":"任务标准不一致","url":"#任务标准不一致","depth":3},{"value":"八、偏好优化会带来哪些副作用","url":"#八偏好优化会带来哪些副作用","depth":2},{"value":"九、KL 约束为什么重要","url":"#九kl-约束为什么重要","depth":2},{"value":"十、离线偏好和在线反馈如何结合","url":"#十离线偏好和在线反馈如何结合","depth":2},{"value":"十一、什么时候选 RLHF，什么时候选 DPO","url":"#十一什么时候选-rlhf什么时候选-dpo","depth":2},{"value":"十二、我的总结：对齐是持续校准，不是一次训练","url":"#十二我的总结对齐是持续校准不是一次训练","depth":2}] Sun, 04 Aug 2024 10:00:00 GMT admin@hansking.cn (HANS) AILLMRLHFDPO对齐