[{"data":1,"prerenderedAt":317},["ShallowReactive",2],{"content-\u002Fposts\u002Frotational-positional-encoding":3,"blog-posts":43},{"id":4,"title":5,"body":6,"categories":30,"date":31,"description":26,"draft":32,"extension":33,"meta":34,"navigation":35,"path":36,"seo":37,"stem":38,"tags":39,"__hash__":42},"posts\u002Fposts\u002Frotational-positional-encoding.md","RoPE",{"type":7,"value":8,"toc":25},"minimark",[9,13,16,19,22],[10,11,12],"p",{},"Attn 本身不表示位置，不包含时序信息。在进入 Attn Block 之前，需要将文本的位置信息编码进 Embedding vector 里。",[10,14,15],{},"早期如 BERT 使用的是绝对位置编码，用一个 Embedding 去表示位置，无法外推，并且还需要额外存储位置信息。相对位置编码，用两个 token 之间的相对距离去编码解决了外推的问题，但实现很麻烦。",[10,17,18],{},"我们知道 Attn 本身就是抽象出 Query、Key、Value，计算 QK 相关性找 V 这个过程，这里用了矩阵乘法。对于矩阵我们有很多 trick。同时旋转 Q、K 两个矩阵，他们矩阵点乘的结果就能实现相对位置编码，而旋转过程中频率的衰减自然展现出了绝对位置的信息。",[10,20,21],{},"RoPE 利用了矩阵的数学特性，漂亮地做好了位置编码这件事。只需要做旋转就可以。空间复杂度 O(1)，算法复杂度都降了很多。把大的 QK 矩阵分块改成一对一对的到二维空间做旋转，这些维度对从高到低不同频率衰减包含在其中。高频负责近距离，中频平衡精度和范围，低频确定远距离位置。外推能力强，没有引入额外的复杂度，非常美丽。",[10,23,24],{},"如果位置信息能从 off 到其他模块，RoPE 这块还能减负。比如使用 Linear Attn，如 Gated DeltaNet，自带很强的位置理解。Attn 就可以把长程的位置理解 off 掉，用 Partial RoPE 保留高频部分，只理解近邻的位置信息即可。",{"title":26,"searchDepth":27,"depth":28,"links":29},"",2,3,[],"技术","2026-06-17 18:31:40",false,"md",{},true,"\u002Fposts\u002Frotational-positional-encoding",{"title":5,"description":26},"posts\u002Frotational-positional-encoding",[40,41,5],"LLMs","Transformer","EOSB_TZPd-C0c5rxD06sK5W8w5LCyJ-TVdZoutLXDlg",[44,53,59,66,71,80,89,95,99,108,117,123,129,137,141,145,155,162,168,172,176,184,191,199,204,210,212,217,222,229,236,243,249,256,264,270,276,282,288,293,302,310],{"path":45,"title":46,"description":47,"date":48,"categories":49,"tags":50,"draft":35},"\u002Fposts\u002Ftransposition-borrowed-chords","平移借用：大调里立刻能用的和弦","把大调整体平移，借用各调的顺阶和弦；有符号平移量 ±2、±3、±5、±7 给出七个近关系调，合计 25 个现成三和弦与 31 个现成七和弦，全部 12 个平移量给 36 与 48。","2026-11-08 16:20:00","音乐",[51,52],"乐理","和弦",{"path":54,"title":55,"description":56,"date":57,"categories":49,"tags":58,"draft":35},"\u002Fposts\u002Fdiatonic-chords-major-system","大调体系下的三和弦和七和弦","以同主音 C 为基准，逐个推出七套音阶的顺阶三和弦与七和弦，标注每个和弦的出处，去重后得到 21 个三和弦与 28 个七和弦。","2026-11-08 15:20:00",[51,52],{"path":60,"title":61,"description":26,"date":62,"categories":63,"tags":64,"draft":35},"\u002Fposts\u002F作息感慨","作息感慨","2026-11-04 07:15:08","生活杂谈",[65],"日常",{"path":67,"title":68,"description":26,"date":69,"categories":26,"tags":70,"draft":35},"\u002Fposts\u002Fgp-tab-测试","gp-tab-测试","2026-10-12 10:51:22",null,{"path":72,"title":73,"description":74,"date":75,"categories":30,"tags":76,"draft":32},"\u002Fposts\u002Fnetwork-clean","怎么让网络变得干净","一个干净的网络，让人省心的使用服务。","2026-10-11 19:40:02",[77,78,79],"VPN","网络","AI",{"path":81,"title":82,"description":83,"date":84,"categories":63,"tags":85,"draft":32},"\u002Fposts\u002Fthe-age-of-hesitation","[搬运] 不敢投入的时代","为什么我们现在越来越不敢投入了。很多时候我们以为的选择，比如你有两个工作offer，现在选择好像是我到底应该考公还是考编。明明这两个选项都不是真正的选项，因为你还没有开始，也不知道最后能不能成。这个时候我们就已经开始犹豫了。","2026-10-10 11:44:12",[86,87,88],"翻译","搬运","心理学",{"path":90,"title":91,"description":92,"date":93,"categories":63,"tags":94,"draft":35},"\u002Fposts\u002Fwhy-happiness-look-like-same","[搬运] 为什么大家的幸福都长一个样？","为什么大家的幸福都长一个样？一种认知被广泛传播后似乎变得不断扁平，这种感觉很奇怪。","2026-10-09 13:57:15",[86,87,88],{"path":96,"title":97,"description":26,"date":98,"categories":63,"tags":70,"draft":35},"\u002Fposts\u002Feverything-is-paid-twice","任何事情都要被消费两次","2026-10-06 12:30:06",{"path":100,"title":101,"description":102,"date":103,"categories":104,"tags":105,"draft":32},"\u002Fposts\u002Fllm","为什么模型上线后，体感像是“降智”了？","权重与 KV cache 的量化都能降低 serving 成本，也可能改变输出；但体感变弱不足以证明服务后来切换了精度。","2026-09-29 21:44:08","技术杂谈",[40,106,107],"Serving","量化",{"path":109,"title":110,"description":111,"date":112,"categories":30,"tags":113,"draft":32},"\u002Fposts\u002Fspinning-up-policy-optimization","[译] 深度强化学习入门（三）：策略优化入门","OpenAI Spinning Up 教程第三部分的全文翻译：推导最简单的策略梯度、log-derivative trick、EGLP 引理、reward-to-go 与基线，配 PyTorch 示例代码。","2026-09-28 17:49:25",[114,115,116,86],"强化学习","RL","Spinning Up",{"path":118,"title":119,"description":120,"date":121,"categories":30,"tags":122,"draft":32},"\u002Fposts\u002Fspinning-up-rl-intro-2","[译] 深度强化学习入门（二）：强化学习算法的分类","OpenAI Spinning Up 第二部分的全文翻译：无模型与基于模型的 RL、策略优化与 Q-learning 的取舍，以及规划、专家迭代和经验增强等方法。","2026-09-28 10:20:21",[114,115,116,86],{"path":124,"title":125,"description":126,"date":127,"categories":30,"tags":128,"draft":32},"\u002Fposts\u002Fspinning-up-rl-intro","[译] 深度强化学习入门（一）：RL 的关键概念","OpenAI Spinning Up 教程第一部分的全文翻译：智能体与环境、状态与观测、动作空间、策略、轨迹、奖励与回报、价值函数、贝尔曼方程、优势函数，附 MDP 形式化。","2026-09-28 09:29:40",[114,115,116,86],{"path":130,"title":131,"description":132,"date":133,"categories":63,"tags":134,"draft":35},"\u002Fposts\u002Fwhen-machines-come-alive","冰箱之后，生活还会怎样改变？","从厨房里的果蝇想到冰箱、移动互联网与 AI 的多轮调用：AGI 已经到来，只是昂贵的智能会怎样改变生活？","2026-09-25 06:36:22",[79,135,136],"随笔","科技",{"path":138,"title":139,"description":26,"date":140,"categories":26,"tags":70,"draft":35},"\u002Fposts\u002Fmathematical-foundations-of-llms-1","LLM的数学基础（1）：","2026-09-19 22:59:55",{"path":142,"title":143,"description":26,"date":144,"categories":26,"tags":70,"draft":35},"\u002Fposts\u002Ftowards-maximize-serving-capatity","如何在 offline bench 里最大化 LLM serving system 的吞吐","2026-09-19 05:45:05",{"path":146,"title":147,"description":148,"date":149,"categories":104,"tags":150,"draft":32},"\u002Fposts\u002Fllm-memory-compute-tradeoff","LLM 的 Memory-Compute Trade-off：从 MHA 到 MLA、Sparse 与 Linear Attention","ds4.1f 上线了。它的架构 novel 在哪，便宜和快又是从哪省出来的。","2026-09-11 12:00:00",[40,151,152,153,154],"Attention","MLA","DeepSeek","长上下文",{"path":156,"title":157,"description":158,"date":159,"categories":104,"tags":160,"draft":35},"\u002Fposts\u002Fmarkdown-extensions","Markdown 扩展语法速查","本站 Markdown 支持的扩展语法演示与速查：admonition 提示块、GitHub 仓库卡片、spoiler 遮罩、ABC 乐谱、GP 乐谱、音频播放器、浏览器可运行代码块。","2026-09-01 15:30:00",[161],"博客",{"path":163,"title":164,"description":165,"date":166,"categories":63,"tags":167,"draft":35},"\u002Fposts","为什么还需要手写博客？","AI这么强，还要古法写博客吗？","2026-09-01 13:17:53",[161],{"path":169,"title":170,"description":26,"date":171,"categories":26,"tags":70,"draft":35},"\u002Fposts\u002Fllm-serving-1","LLM Serving（一）:以在H200上部署GLM-5.2为例","2026-07-21 06:15:44",{"path":173,"title":174,"description":26,"date":175,"categories":49,"tags":70,"draft":32},"\u002Fposts\u002Finteresting-music-rhythm","有趣的音乐节奏","2026-07-16 12:47:24",{"path":177,"title":178,"description":179,"date":180,"categories":30,"tags":181,"draft":32},"\u002Fposts\u002Ftowards-monosemanticity","LLM 可解释性（一）：SAE与单语义","读 Anthropic《Towards Monosemanticity》。从难以解释的神经元出发, 讲清superposition为什么让单个神经元混成一团, SAE怎么把多语义的激活分解成单语义特征。","2026-07-12 22:00:00",[40,182,183],"可解释性","SAE",{"path":185,"title":186,"description":187,"date":188,"categories":63,"tags":189,"draft":35},"\u002Fposts\u002Fpiketty-ai-china-k-shaped","用皮凯蒂的镜头看 AI 与中国 K 型社会","一次凌晨的对话整理。从 r>g 出发，看 AI 时代的资本分配、中国的世袭化趋势、体制内的二元劳动力市场，以及一个普通 00 后在一线打拼的真实牌面。","2026-06-24 03:00:00",[135,190,79],"经济",{"path":192,"title":193,"description":26,"date":194,"categories":26,"tags":195,"draft":35},"\u002Fposts\u002Fclaude-code-compact-behavior","cluade code compact 行为","2026-06-23 20:17:42",[196,197,198],"Agent","AI Agent","Claude Code",{"path":200,"title":201,"description":26,"date":202,"categories":30,"tags":203,"draft":35},"\u002Fposts\u002Fclaude-code-settings","Claude Code 配置分享","2026-06-23 14:22:12",[40,197,196,198],{"path":205,"title":206,"description":207,"date":208,"categories":104,"tags":209,"draft":32},"\u002Fposts\u002Fmy-thoughts-on-the-ai-industry","简单聊聊对 AI Agent 行业的理解","写这篇文章的初衷，是我发觉逐渐忘了前两个月的 AI\u002FLLMs 是什么样的能力和情况了。意识中似乎某些在突然加速，但理性和直觉告诉我这不对，作为行内人，我知道这是一个渐进的、累积的过程。我想自己回忆一下，Agent的发展。","2026-06-20 13:14:25",[40,197,196],{"path":36,"title":5,"description":26,"date":31,"categories":30,"tags":211,"draft":32},[40,41,5],{"path":213,"title":214,"description":26,"date":215,"categories":30,"tags":216,"draft":32},"\u002Fposts\u002Fintroduction-to-large-language-model","大语言模型入门基础","2026-06-16 22:36:11",[41,40],{"path":218,"title":219,"description":26,"date":220,"categories":104,"tags":221,"draft":35},"\u002Fposts\u002Fcoding-agent","关于 Coding Agent 使用的遐思","2026-06-03 17:54:20",[196],{"path":223,"title":224,"description":225,"date":226,"categories":30,"tags":227,"draft":35},"\u002Fposts\u002Fabout-ssh-usage","关于SSH的使用——客户端篇","介绍SSH客户端的用法。","2025-06-17 02:44:57",[228],"工具",{"path":230,"title":231,"description":232,"date":233,"categories":104,"tags":234,"draft":32},"\u002Fposts\u002Fopensourcesoftwarestarttutorial","如何开始一个开源项目？","浅谈我对开源项目的认知。","2025-04-17 18:00:00",[235],"开源",{"path":237,"title":238,"description":239,"date":240,"categories":241,"tags":242,"draft":35},"\u002Fposts\u002Fmotivation","渴求动力","有意识的生活并主动地行动会使得生活更美好。","2025-03-05 00:44:57","生活-情感",[135],{"path":244,"title":245,"description":246,"date":247,"categories":241,"tags":248,"draft":35},"\u002Fposts\u002F2024summary","2024年终总结","啊，痛彻的时光；啊，悲怆的人！","2025-01-01 01:00:00",[135],{"path":250,"title":251,"description":252,"date":253,"categories":30,"tags":254,"draft":35},"\u002Fposts\u002Fvps-crash-startup","快速搭建个人VPS","如何快速搭建个人VPS。","2024-12-31 19:00:00",[255],"VPS",{"path":257,"title":258,"description":259,"date":260,"categories":30,"tags":261,"draft":35},"\u002Fposts\u002Fdata-label-software","AI数据（自动）标注软件概述","对市面上目前的AI数据（自动）标注软件进行调研。","2024-12-20 14:00:00",[262,263],"数据标注","深度学习",{"path":265,"title":266,"description":267,"date":268,"categories":241,"tags":269,"draft":32},"\u002Fposts\u002Fself-destruction","生活的崩坏与自我毁灭","是什么在牵引着我？我的行为，我的思想？我的感受？我所处的地方？它们在何处？","2024-12-19 12:00:00",[135],{"path":271,"title":272,"description":273,"date":274,"categories":30,"tags":275,"draft":35},"\u002Fposts\u002Fcs194-196_llmagents","LLM Agents 课程笔记","UCB CS294\u002F194-196 Large Language Model Agents 的课程笔记。","2024-11-30 20:00:00",[196],{"path":277,"title":278,"description":279,"date":280,"categories":241,"tags":281,"draft":35},"\u002Fposts\u002Fwhatsyourworth","价值、位置","你们的生存究竟有何价值？如果毫无价值，你们究竟为何存在？","2024-11-29 02:00:00",[135],{"path":283,"title":284,"description":285,"date":286,"categories":241,"tags":287,"draft":32},"\u002Fposts\u002Fblogrelifever2","重启博客v2","忙里偷闲学习前端，还是决定结合想法手搓一个自己的博客。","2024-11-25 00:00:00",[161],{"path":289,"title":290,"description":291,"date":292,"categories":241,"tags":70,"draft":35},"\u002Fposts\u002Fblogrelife","重启博客","在我的表达欲与现实所需(问题解决记录)驱动下，我决定重写博客。","2024-10-21 20:44:57",{"path":294,"title":295,"description":296,"date":297,"categories":63,"tags":298,"draft":32},"\u002Fposts\u002Fsurvival-welcome-to-university","生存手册——欢迎来到大学","大学同时生产三样东西：可验证技能、认证信号和驯化资本。你付了四年的钱，真正拿到手的是哪一样，取决于你毕业时要进的那个市场在为什么付费。","2024-10-12 20:00:00",[299,300,301],"大学","求职","方法论",{"path":303,"title":304,"description":305,"date":306,"categories":241,"tags":307,"draft":32},"\u002Fposts\u002Frevival","240205 复活","好吧，断更了许久，我又终于回来了，我又终于回来了。","2024-02-05 20:00:00",[135,308,309],"生活","返乡",{"path":311,"title":312,"description":313,"date":314,"categories":241,"tags":315,"draft":35},"\u002Fposts\u002Fclouds","「230717」云","我已经很久忘记了不我已经很少把自己的想法在lofter上发布的。","2023-07-17 20:00:00",[135,316],"散文",1795115926097]