https://www.hansking.cn/blog A posts created with Next.js and Tailwind.css en-us admin@hansking.cn (HANS) admin@hansking.cn (HANS) Sun, 29 Sep 2024 10:00:00 GMT https://www.hansking.cn/blog/2026/RAG-shu-ju-guan-dao-jie-xi-PDF-wang-ye-yu-biao-ge https://www.hansking.cn/blog/2026/RAG-shu-ju-guan-dao-jie-xi-PDF-wang-ye-yu-biao-ge [{"value":"RAG 数据管道：解析 PDF、网页与表格","url":"#rag-数据管道解析-pdf网页与表格","depth":1},{"value":"一、先承认：不同格式不是同一种文档","url":"#一先承认不同格式不是同一种文档","depth":2},{"value":"二、PDF：先处理布局，再处理文本","url":"#二pdf先处理布局再处理文本","depth":2},{"value":"三、网页：正文只是页面的一小部分","url":"#三网页正文只是页面的一小部分","depth":2},{"value":"四、表格：关系比自然语言更重要","url":"#四表格关系比自然语言更重要","depth":2},{"value":"五、切分：不是越小越好，也不是越大越完整","url":"#五切分不是越小越好也不是越大越完整","depth":2},{"value":"六、清洗要克制：噪声和信息不能一锅端","url":"#六清洗要克制噪声和信息不能一锅端","depth":2},{"value":"七、去重和版本：避免知识库里有十个“同一份答案”","url":"#七去重和版本避免知识库里有十个同一份答案","depth":2},{"value":"八、权限元数据必须跟着每个 chunk 走","url":"#八权限元数据必须跟着每个-chunk-走","depth":2},{"value":"九、质量检查要在入库时就发生","url":"#九质量检查要在入库时就发生","depth":2},{"value":"十、增量处理：文档变化时只做必要工作","url":"#十增量处理文档变化时只做必要工作","depth":2},{"value":"十一、如何评测管道是否真的变好了","url":"#十一如何评测管道是否真的变好了","depth":2},{"value":"十二、我的总结：RAG 的第一模型是数据管道","url":"#十二我的总结rag-的第一模型是数据管道","depth":2}] Sun, 29 Sep 2024 10:00:00 GMT admin@hansking.cn (HANS) AILLMRAG数据工程文档解析 https://www.hansking.cn/blog/2026/SFT-zhi-ling-wei-tiao-shu-ju-ge-shi-qing-xi-yu-xun-lian https://www.hansking.cn/blog/2026/SFT-zhi-ling-wei-tiao-shu-ju-ge-shi-qing-xi-yu-xun-lian [{"value":"SFT 指令微调：数据格式、清洗与训练","url":"#sft-指令微调数据格式清洗与训练","depth":1},{"value":"一、先判断问题是不是真的需要 SFT","url":"#一先判断问题是不是真的需要-sft","depth":2},{"value":"二、SFT 数据的基本形式","url":"#二sft-数据的基本形式","depth":2},{"value":"三、输入输出边界必须一致","url":"#三输入输出边界必须一致","depth":2},{"value":"四、清洗：模型会把脏数据学得很认真","url":"#四清洗模型会把脏数据学得很认真","depth":2},{"value":"去重","url":"#去重","depth":3},{"value":"冲突检查","url":"#冲突检查","depth":3},{"value":"敏感信息","url":"#敏感信息","depth":3},{"value":"模型生成数据","url":"#模型生成数据","depth":3},{"value":"五、数据划分要防止“换句话说的泄漏”","url":"#五数据划分要防止换句话说的泄漏","depth":2},{"value":"六、长度和截断要在训练前算清楚","url":"#六长度和截断要在训练前算清楚","depth":2},{"value":"七、loss 到底应该算在哪些 Token 上","url":"#七loss-到底应该算在哪些-token-上","depth":2},{"value":"八、一个简化的训练配置","url":"#八一个简化的训练配置","depth":2},{"value":"九、训练过程中看什么","url":"#九训练过程中看什么","depth":2},{"value":"十、训练完成后先做离线回归","url":"#十训练完成后先做离线回归","depth":2},{"value":"十一、部署时不要忘记训练模板","url":"#十一部署时不要忘记训练模板","depth":2},{"value":"十二、什么时候训练算成功","url":"#十二什么时候训练算成功","depth":2},{"value":"十三、我的总结：SFT 是数据工程，不只是训练工程","url":"#十三我的总结sft-是数据工程不只是训练工程","depth":2}] Sun, 21 Jul 2024 10:00:00 GMT admin@hansking.cn (HANS) AILLMSFT微调数据工程