https://www.hansking.cn/blog A posts created with Next.js and Tailwind.css en-us admin@hansking.cn (HANS) admin@hansking.cn (HANS) Sun, 09 Jun 2024 10:00:00 GMT https://www.hansking.cn/blog/2026/MoE-de-lu-you-fu-zai-jun-heng-yu-xun-lian-nan-dian https://www.hansking.cn/blog/2026/MoE-de-lu-you-fu-zai-jun-heng-yu-xun-lian-nan-dian [{"value":"MoE 的路由、负载均衡与训练难点","url":"#moe-的路由负载均衡与训练难点","depth":1},{"value":"一、普通 Transformer 为什么需要专家","url":"#一普通-transformer-为什么需要专家","depth":2},{"value":"二、路由器在做什么","url":"#二路由器在做什么","depth":2},{"value":"三、Top-k 为什么既有效又麻烦","url":"#三top-k-为什么既有效又麻烦","depth":2},{"value":"四、负载均衡是 MoE 的核心难题","url":"#四负载均衡是-moe-的核心难题","depth":2},{"value":"五、容量因子和 Token dropping","url":"#五容量因子和-token-dropping","depth":2},{"value":"六、专家为什么会学出不同能力","url":"#六专家为什么会学出不同能力","depth":2},{"value":"七、训练时的通信可能比计算更贵","url":"#七训练时的通信可能比计算更贵","depth":2},{"value":"八、训练稳定性需要多种信号共同判断","url":"#八训练稳定性需要多种信号共同判断","depth":2},{"value":"九、推理部署和训练不是同一笔账","url":"#九推理部署和训练不是同一笔账","depth":2},{"value":"十、如何判断 MoE 是否值得使用","url":"#十如何判断-moe-是否值得使用","depth":2},{"value":"十一、我的总结：MoE 是模型里的调度系统","url":"#十一我的总结moe-是模型里的调度系统","depth":2}] Sun, 09 Jun 2024 10:00:00 GMT admin@hansking.cn (HANS) AILLMMoE稀疏专家深度学习