https://www.hansking.cn/blog A posts created with Next.js and Tailwind.css en-us admin@hansking.cn (HANS) admin@hansking.cn (HANS) Sun, 18 Aug 2024 10:00:00 GMT https://www.hansking.cn/blog/2026/DPO-shi-zhan-xun-lian-yi-ge-geng-fu-he-ye-wu-pian-hao-de-mo-xing https://www.hansking.cn/blog/2026/DPO-shi-zhan-xun-lian-yi-ge-geng-fu-he-ye-wu-pian-hao-de-mo-xing [{"value":"DPO 实战：训练一个更符合业务偏好的模型","url":"#dpo-实战训练一个更符合业务偏好的模型","depth":1},{"value":"一、先回顾 SFT 和 RLHF 的位置","url":"#一先回顾-sft-和-rlhf-的位置","depth":2},{"value":"二、偏好数据格式很简单，数据含义不简单","url":"#二偏好数据格式很简单数据含义不简单","depth":2},{"value":"三、偏好标注最容易出现什么问题","url":"#三偏好标注最容易出现什么问题","depth":2},{"value":"把长度当质量","url":"#把长度当质量","depth":3},{"value":"把语气当事实","url":"#把语气当事实","depth":3},{"value":"偏好标准互相冲突","url":"#偏好标准互相冲突","depth":3},{"value":"负样本太差","url":"#负样本太差","depth":3},{"value":"四、DPO 的损失直觉","url":"#四dpo-的损失直觉","depth":2},{"value":"五、训练前先处理 Token 和截断","url":"#五训练前先处理-token-和截断","depth":2},{"value":"六、一个简化的训练流程","url":"#六一个简化的训练流程","depth":2},{"value":"七、LoRA 能降低成本，但不能降低评测要求","url":"#七lora-能降低成本但不能降低评测要求","depth":2},{"value":"八、DPO 训练中要盯哪些信号","url":"#八dpo-训练中要盯哪些信号","depth":2},{"value":"九、训练后评测必须包含“不可偏好的”样本","url":"#九训练后评测必须包含不可偏好的样本","depth":2},{"value":"十、如何处理矛盾偏好","url":"#十如何处理矛盾偏好","depth":2},{"value":"十一、什么时候 DPO 不适合","url":"#十一什么时候-dpo-不适合","depth":2},{"value":"十二、我的总结：偏好优化是在教模型“更像你的团队”","url":"#十二我的总结偏好优化是在教模型更像你的团队","depth":2}] Sun, 18 Aug 2024 10:00:00 GMT admin@hansking.cn (HANS) AILLMDPO微调偏好优化 https://www.hansking.cn/blog/2026pian-hao-you-hua-cong-RLHF-dao-DPO https://www.hansking.cn/blog/2026pian-hao-you-hua-cong-RLHF-dao-DPO [{"value":"偏好优化：从 RLHF 到 DPO","url":"#偏好优化从-rlhf-到-dpo","depth":1},{"value":"一、预训练和 SFT 解决了什么","url":"#一预训练和-sft-解决了什么","depth":2},{"value":"二、偏好数据把绝对打分改成相对比较","url":"#二偏好数据把绝对打分改成相对比较","depth":2},{"value":"三、奖励模型：把偏好变成一个分数","url":"#三奖励模型把偏好变成一个分数","depth":2},{"value":"四、PPO 为什么被用于 RLHF","url":"#四ppo-为什么被用于-rlhf","depth":2},{"value":"五、DPO 的直觉：直接让模型偏爱 chosen","url":"#五dpo-的直觉直接让模型偏爱-chosen","depth":2},{"value":"六、三种路线放在一起看","url":"#六三种路线放在一起看","depth":2},{"value":"七、为什么数据质量比算法名字更重要","url":"#七为什么数据质量比算法名字更重要","depth":2},{"value":"偏好差异不清楚","url":"#偏好差异不清楚","depth":3},{"value":"chosen 只是更长","url":"#chosen-只是更长","depth":3},{"value":"rejected 过于糟糕","url":"#rejected-过于糟糕","depth":3},{"value":"任务标准不一致","url":"#任务标准不一致","depth":3},{"value":"八、偏好优化会带来哪些副作用","url":"#八偏好优化会带来哪些副作用","depth":2},{"value":"九、KL 约束为什么重要","url":"#九kl-约束为什么重要","depth":2},{"value":"十、离线偏好和在线反馈如何结合","url":"#十离线偏好和在线反馈如何结合","depth":2},{"value":"十一、什么时候选 RLHF，什么时候选 DPO","url":"#十一什么时候选-rlhf什么时候选-dpo","depth":2},{"value":"十二、我的总结：对齐是持续校准，不是一次训练","url":"#十二我的总结对齐是持续校准不是一次训练","depth":2}] Sun, 04 Aug 2024 10:00:00 GMT admin@hansking.cn (HANS) AILLMRLHFDPO对齐