[{"data":1,"prerenderedAt":28963},["ShallowReactive",2],{"content-\u002Fposts\u002Fllm-memory-compute-tradeoff":3,"blog-posts":28693},{"id":4,"title":5,"body":6,"categories":28679,"date":28680,"description":28681,"draft":28682,"extension":28683,"meta":28684,"navigation":26176,"path":28685,"seo":28686,"stem":28687,"tags":28688,"__hash__":28692},"posts\u002Fposts\u002Fllm-memory-compute-tradeoff.md","LLM 的 Memory-Compute Trade-off：从 MHA 到 MLA、Sparse 与 Linear Attention",{"type":7,"value":8,"toc":28661},"minimark",[9,17,20,25,594,967,2046,3045,3495,3690,3810,3943,3946,3949,3953,3967,3974,3981,3984,4284,4774,5022,5026,5149,5152,5155,5217,5296,5469,5551,6014,6226,6229,6335,6346,6429,6434,6441,6445,6448,6552,7178,7324,8131,8605,8608,9262,9392,10166,11148,11374,11377,11380,12033,12217,13146,13512,13781,14172,14177,14644,15249,15252,15436,15439,15605,15608,16457,16491,16826,16893,17231,17262,17689,17725,17731,17734,17849,17852,17855,17888,18229,18289,18573,18575,18952,18955,18958,18972,19282,19288,19294,19297,19300,19477,19649,19652,19721,19727,19730,20012,20076,20080,20085,20370,20375,20378,20440,20443,20463,20466,20469,20610,20678,20755,20895,20899,21045,21048,21141,21144,21148,21155,21502,21505,21800,22027,22081,22322,22444,22451,22455,22461,22465,22468,22478,22481,22487,22502,22505,22508,22557,22658,22662,22912,22917,23011,23105,23258,23261,23370,23487,23496,23499,23562,23632,23939,23942,23951,23977,24067,24145,24177,24201,24204,24207,24210,24583,24592,24599,24606,24609,24656,24659,24662,24818,24821,24928,25041,25232,25235,25241,25244,25315,25485,25488,25525,25531,25534,25537,25540,25662,25668,25671,25834,25837,25840,25845,25848,25851,26100,26103,26419,26635,26798,26881,26884,27049,27054,27276,27339,27395,27401,27404,27407,27471,27474,27480,27647,27653,27758,27761,27902,27905,28097,28103,28106,28109,28116,28213,28216,28222,28225,28229,28232,28241,28250,28253,28256,28263,28324,28335,28365,28368,28373,28376,28379,28452,28455,28570,28573,28599,28605,28608,28611,28614,28617,28620,28626,28633,28636,28642,28648,28654,28657],[10,11,12,13],"p",{},"GPT6 出来之后，体感上很多人都很焦虑迷茫，真的到了 AGI time，接下来该做什么呢？虽然说 Astra 用下来，在一些 case 上还是会 overfit，但真的已经能胜任大多数人完成的任务了（在数字世界里）。这两天 ds4.1f 正式上线了，如此 novel 的架构让我感到新奇，模型架构与硬件的 co-design 达到一定水平的杰作，让我们来看一下吧。",[14,15,16],"strong",{},"这篇由 AI 整理。",[10,18,19],{},"从 MHA 出发，一路到 Linear Attention，看每一代分别在砍哪一项成本。",[21,22,24],"h2",{"id":23},"mha-与-kv-cache","MHA 与 KV Cache",[10,26,27,28,277,278,413,414,444,445,491,492,593],{},"MHA（Multi-Head Attention）是这一切的起点，2017 年《Attention Is All You Need》里提出来的。数学上它就是多个独立的单头注意力拼起来。假设输入向量序列是 ",[29,30,33,93],"span",{"className":31},[32],"katex",[29,34,37],{"className":35},[36],"katex-mathml",[38,39,41],"math",{"xmlns":40},"http:\u002F\u002Fwww.w3.org\u002F1998\u002FMath\u002FMathML",[42,43,44,88],"semantics",{},[45,46,47,58,63,70,72,75,79,81],"mrow",{},[48,49,50,54],"msub",{},[51,52,53],"mi",{},"x",[55,56,57],"mn",{},"1",[59,60,62],"mo",{"separator":61},"true",",",[48,64,65,67],{},[51,66,53],{},[55,68,69],{},"2",[59,71,62],{"separator":61},[59,73,74],{},"⋯",[76,77,78],"mtext",{}," ",[59,80,62],{"separator":61},[48,82,83,85],{},[51,84,53],{},[51,86,87],{},"l",[89,90,92],"annotation",{"encoding":91},"application\u002Fx-tex","x_1, x_2, \\cdots, x_l",[29,94,97],{"className":95,"ariaHidden":61},[96],"katex-html",[29,98,101,106,164,168,173,213,216,219,223,226,229,232,235],{"className":99},[100],"base",[29,102],{"className":103,"style":105},[104],"strut","height:0.625em;vertical-align:-0.1944em;",[29,107,110,114],{"className":108},[109],"mord",[29,111,53],{"className":112},[109,113],"mathnormal",[29,115,118],{"className":116},[117],"msupsub",[29,119,123,155],{"className":120},[121,122],"vlist-t","vlist-t2",[29,124,127,150],{"className":125},[126],"vlist-r",[29,128,132],{"className":129,"style":131},[130],"vlist","height:0.3011em;",[29,133,135,140],{"style":134},"top:-2.55em;margin-left:0em;margin-right:0.05em;",[29,136],{"className":137,"style":139},[138],"pstrut","height:2.7em;",[29,141,147],{"className":142},[143,144,145,146],"sizing","reset-size6","size3","mtight",[29,148,57],{"className":149},[109,146],[29,151,154],{"className":152},[153],"vlist-s","​",[29,156,158],{"className":157},[126],[29,159,162],{"className":160,"style":161},[130],"height:0.15em;",[29,163],{},[29,165,62],{"className":166},[167],"mpunct",[29,169],{"className":170,"style":172},[171],"mspace","margin-right:0.1667em;",[29,174,176,179],{"className":175},[109],[29,177,53],{"className":178},[109,113],[29,180,182],{"className":181},[117],[29,183,185,205],{"className":184},[121,122],[29,186,188,202],{"className":187},[126],[29,189,191],{"className":190,"style":131},[130],[29,192,193,196],{"style":134},[29,194],{"className":195,"style":139},[138],[29,197,199],{"className":198},[143,144,145,146],[29,200,69],{"className":201},[109,146],[29,203,154],{"className":204},[153],[29,206,208],{"className":207},[126],[29,209,211],{"className":210,"style":161},[130],[29,212],{},[29,214,62],{"className":215},[167],[29,217],{"className":218,"style":172},[171],[29,220,74],{"className":221},[222],"minner",[29,224],{"className":225,"style":172},[171],[29,227],{"className":228,"style":172},[171],[29,230,62],{"className":231},[167],[29,233],{"className":234,"style":172},[171],[29,236,238,241],{"className":237},[109],[29,239,53],{"className":240},[109,113],[29,242,244],{"className":243},[117],[29,245,247,269],{"className":246},[121,122],[29,248,250,266],{"className":249},[126],[29,251,254],{"className":252,"style":253},[130],"height:0.3361em;",[29,255,256,259],{"style":134},[29,257],{"className":258,"style":139},[138],[29,260,262],{"className":261},[143,144,145,146],[29,263,87],{"className":264,"style":265},[109,113,146],"margin-right:0.0197em;",[29,267,154],{"className":268},[153],[29,270,272],{"className":271},[126],[29,273,275],{"className":274,"style":161},[130],[29,276],{},"，其中 ",[29,279,281,313],{"className":280},[32],[29,282,284],{"className":283},[36],[38,285,286],{"xmlns":40},[42,287,288,310],{},[45,289,290,297,300],{},[48,291,292,294],{},[51,293,53],{},[51,295,296],{},"i",[59,298,299],{},"∈",[301,302,303,307],"msup",{},[51,304,306],{"mathvariant":305},"double-struck","R",[51,308,309],{},"d",[89,311,312],{"encoding":91},"x_i \\in \\mathbb{R}^d",[29,314,316,375],{"className":315,"ariaHidden":61},[96],[29,317,319,323,364,368,372],{"className":318},[100],[29,320],{"className":321,"style":322},[104],"height:0.6891em;vertical-align:-0.15em;",[29,324,326,329],{"className":325},[109],[29,327,53],{"className":328},[109,113],[29,330,332],{"className":331},[117],[29,333,335,356],{"className":334},[121,122],[29,336,338,353],{"className":337},[126],[29,339,342],{"className":340,"style":341},[130],"height:0.3117em;",[29,343,344,347],{"style":134},[29,345],{"className":346,"style":139},[138],[29,348,350],{"className":349},[143,144,145,146],[29,351,296],{"className":352},[109,113,146],[29,354,154],{"className":355},[153],[29,357,359],{"className":358},[126],[29,360,362],{"className":361,"style":161},[130],[29,363],{},[29,365],{"className":366,"style":367},[171],"margin-right:0.2778em;",[29,369,299],{"className":370},[371],"mrel",[29,373],{"className":374,"style":367},[171],[29,376,378,382],{"className":377},[100],[29,379],{"className":380,"style":381},[104],"height:0.8491em;",[29,383,385,389],{"className":384},[109],[29,386,306],{"className":387},[109,388],"mathbb",[29,390,392],{"className":391},[117],[29,393,395],{"className":394},[121],[29,396,398],{"className":397},[126],[29,399,401],{"className":400,"style":381},[130],[29,402,404,407],{"style":403},"top:-3.063em;margin-right:0.05em;",[29,405],{"className":406,"style":139},[138],[29,408,410],{"className":409},[143,144,145,146],[29,411,309],{"className":412},[109,113,146],"。",[29,415,417,431],{"className":416},[32],[29,418,420],{"className":419},[36],[38,421,422],{"xmlns":40},[42,423,424,429],{},[45,425,426],{},[51,427,428],{},"h",[89,430,428],{"encoding":91},[29,432,434],{"className":433,"ariaHidden":61},[96],[29,435,437,441],{"className":436},[100],[29,438],{"className":439,"style":440},[104],"height:0.6944em;",[29,442,428],{"className":443},[109,113]," 是 head 数，",[29,446,448,470],{"className":447},[32],[29,449,451],{"className":450},[36],[38,452,453],{"xmlns":40},[42,454,455,467],{},[45,456,457,461,464],{},[59,458,460],{"stretchy":459},"false","(",[51,462,463],{},"s",[59,465,466],{"stretchy":459},")",[89,468,469],{"encoding":91},"(s)",[29,471,473],{"className":472,"ariaHidden":61},[96],[29,474,476,480,484,487],{"className":475},[100],[29,477],{"className":478,"style":479},[104],"height:1em;vertical-align:-0.25em;",[29,481,460],{"className":482},[483],"mopen",[29,485,463],{"className":486},[109,113],[29,488,466],{"className":489},[490],"mclose"," 是 head 的下标，",[29,493,495,528],{"className":494},[32],[29,496,498],{"className":497},[36],[38,499,500],{"xmlns":40},[42,501,502,525],{},[45,503,504,506,509,511,513,515,517,519,521,523],{},[51,505,463],{},[59,507,508],{},"=",[55,510,57],{},[59,512,62],{"separator":61},[55,514,69],{},[59,516,62],{"separator":61},[59,518,74],{},[76,520,78],{},[59,522,62],{"separator":61},[51,524,428],{},[89,526,527],{"encoding":91},"s = 1, 2, \\cdots, h",[29,529,531,550],{"className":530,"ariaHidden":61},[96],[29,532,534,538,541,544,547],{"className":533},[100],[29,535],{"className":536,"style":537},[104],"height:0.4306em;",[29,539,463],{"className":540},[109,113],[29,542],{"className":543,"style":367},[171],[29,545,508],{"className":546},[371],[29,548],{"className":549,"style":367},[171],[29,551,553,557,560,563,566,569,572,575,578,581,584,587,590],{"className":552},[100],[29,554],{"className":555,"style":556},[104],"height:0.8889em;vertical-align:-0.1944em;",[29,558,57],{"className":559},[109],[29,561,62],{"className":562},[167],[29,564],{"className":565,"style":172},[171],[29,567,69],{"className":568},[109],[29,570,62],{"className":571},[167],[29,573],{"className":574,"style":172},[171],[29,576,74],{"className":577},[222],[29,579],{"className":580,"style":172},[171],[29,582],{"className":583,"style":172},[171],[29,585,62],{"className":586},[167],[29,588],{"className":589,"style":172},[171],[29,591,428],{"className":592},[109,113],"。那么：",[29,595,598],{"className":596},[597],"katex-display",[29,599,601,683],{"className":600},[32],[29,602,604],{"className":603},[36],[38,605,607],{"xmlns":40,"display":606},"block",[42,608,609,680],{},[45,610,611,619,621,624,639,641,655,657,659,661,663,677],{},[48,612,613,616],{},[51,614,615],{},"o",[51,617,618],{},"t",[59,620,508],{},[59,622,623],{"stretchy":459},"[",[625,626,627,629,631],"msubsup",{},[51,628,615],{},[51,630,618],{},[45,632,633,635,637],{},[59,634,460],{"stretchy":459},[55,636,57],{},[59,638,466],{"stretchy":459},[59,640,62],{"separator":61},[625,642,643,645,647],{},[51,644,615],{},[51,646,618],{},[45,648,649,651,653],{},[59,650,460],{"stretchy":459},[55,652,69],{},[59,654,466],{"stretchy":459},[59,656,62],{"separator":61},[59,658,74],{},[76,660,78],{},[59,662,62],{"separator":61},[625,664,665,667,669],{},[51,666,615],{},[51,668,618],{},[45,670,671,673,675],{},[59,672,460],{"stretchy":459},[51,674,428],{},[59,676,466],{"stretchy":459},[59,678,679],{"stretchy":459},"]",[89,681,682],{"encoding":91},"o_t = [o_t^{(1)}, o_t^{(2)}, \\cdots, o_t^{(h)}]",[29,684,686,743],{"className":685,"ariaHidden":61},[96],[29,687,689,693,734,737,740],{"className":688},[100],[29,690],{"className":691,"style":692},[104],"height:0.5806em;vertical-align:-0.15em;",[29,694,696,699],{"className":695},[109],[29,697,615],{"className":698},[109,113],[29,700,702],{"className":701},[117],[29,703,705,726],{"className":704},[121,122],[29,706,708,723],{"className":707},[126],[29,709,712],{"className":710,"style":711},[130],"height:0.2806em;",[29,713,714,717],{"style":134},[29,715],{"className":716,"style":139},[138],[29,718,720],{"className":719},[143,144,145,146],[29,721,618],{"className":722},[109,113,146],[29,724,154],{"className":725},[153],[29,727,729],{"className":728},[126],[29,730,732],{"className":731,"style":161},[130],[29,733],{},[29,735],{"className":736,"style":367},[171],[29,738,508],{"className":739},[371],[29,741],{"className":742,"style":367},[171],[29,744,746,750,753,817,820,823,883,886,889,892,895,898,901,904,964],{"className":745},[100],[29,747],{"className":748,"style":749},[104],"height:1.2948em;vertical-align:-0.25em;",[29,751,623],{"className":752},[483],[29,754,756,759],{"className":755},[109],[29,757,615],{"className":758},[109,113],[29,760,762],{"className":761},[117],[29,763,765,808],{"className":764},[121,122],[29,766,768,805],{"className":767},[126],[29,769,772,784],{"className":770,"style":771},[130],"height:1.0448em;",[29,773,775,778],{"style":774},"top:-2.4542em;margin-left:0em;margin-right:0.05em;",[29,776],{"className":777,"style":139},[138],[29,779,781],{"className":780},[143,144,145,146],[29,782,618],{"className":783},[109,113,146],[29,785,787,790],{"style":786},"top:-3.2198em;margin-right:0.05em;",[29,788],{"className":789,"style":139},[138],[29,791,793],{"className":792},[143,144,145,146],[29,794,796,799,802],{"className":795},[109,146],[29,797,460],{"className":798},[483,146],[29,800,57],{"className":801},[109,146],[29,803,466],{"className":804},[490,146],[29,806,154],{"className":807},[153],[29,809,811],{"className":810},[126],[29,812,815],{"className":813,"style":814},[130],"height:0.2458em;",[29,816],{},[29,818,62],{"className":819},[167],[29,821],{"className":822,"style":172},[171],[29,824,826,829],{"className":825},[109],[29,827,615],{"className":828},[109,113],[29,830,832],{"className":831},[117],[29,833,835,875],{"className":834},[121,122],[29,836,838,872],{"className":837},[126],[29,839,841,852],{"className":840,"style":771},[130],[29,842,843,846],{"style":774},[29,844],{"className":845,"style":139},[138],[29,847,849],{"className":848},[143,144,145,146],[29,850,618],{"className":851},[109,113,146],[29,853,854,857],{"style":786},[29,855],{"className":856,"style":139},[138],[29,858,860],{"className":859},[143,144,145,146],[29,861,863,866,869],{"className":862},[109,146],[29,864,460],{"className":865},[483,146],[29,867,69],{"className":868},[109,146],[29,870,466],{"className":871},[490,146],[29,873,154],{"className":874},[153],[29,876,878],{"className":877},[126],[29,879,881],{"className":880,"style":814},[130],[29,882],{},[29,884,62],{"className":885},[167],[29,887],{"className":888,"style":172},[171],[29,890,74],{"className":891},[222],[29,893],{"className":894,"style":172},[171],[29,896],{"className":897,"style":172},[171],[29,899,62],{"className":900},[167],[29,902],{"className":903,"style":172},[171],[29,905,907,910],{"className":906},[109],[29,908,615],{"className":909},[109,113],[29,911,913],{"className":912},[117],[29,914,916,956],{"className":915},[121,122],[29,917,919,953],{"className":918},[126],[29,920,922,933],{"className":921,"style":771},[130],[29,923,924,927],{"style":774},[29,925],{"className":926,"style":139},[138],[29,928,930],{"className":929},[143,144,145,146],[29,931,618],{"className":932},[109,113,146],[29,934,935,938],{"style":786},[29,936],{"className":937,"style":139},[138],[29,939,941],{"className":940},[143,144,145,146],[29,942,944,947,950],{"className":943},[109,146],[29,945,460],{"className":946},[483,146],[29,948,428],{"className":949},[109,113,146],[29,951,466],{"className":952},[490,146],[29,954,154],{"className":955},[153],[29,957,959],{"className":958},[126],[29,960,962],{"className":961,"style":814},[130],[29,963],{},[29,965,679],{"className":966},[490],[29,968,970],{"className":969},[597],[29,971,973,1216],{"className":972},[32],[29,974,976],{"className":975},[36],[38,977,978],{"xmlns":40,"display":606},[42,979,980,1213],{},[45,981,982,996,998,1022,1024,1039,1041,1061,1063,1082,1084,1087],{},[625,983,984,986,988],{},[51,985,615],{},[51,987,618],{},[45,989,990,992,994],{},[59,991,460],{"stretchy":459},[51,993,463],{},[59,995,466],{"stretchy":459},[59,997,508],{},[45,999,1000,1004,1006,1008,1011,1014,1016,1018,1020],{},[51,1001,1003],{"mathvariant":1002},"normal","A",[51,1005,618],{"mathvariant":1002},[51,1007,618],{"mathvariant":1002},[51,1009,1010],{"mathvariant":1002},"e",[51,1012,1013],{"mathvariant":1002},"n",[51,1015,618],{"mathvariant":1002},[51,1017,296],{"mathvariant":1002},[51,1019,615],{"mathvariant":1002},[51,1021,1013],{"mathvariant":1002},[59,1023,460],{"stretchy":459},[625,1025,1026,1029,1031],{},[51,1027,1028],{},"q",[51,1030,618],{},[45,1032,1033,1035,1037],{},[59,1034,460],{"stretchy":459},[51,1036,463],{},[59,1038,466],{"stretchy":459},[59,1040,62],{"separator":61},[625,1042,1043,1046,1053],{},[51,1044,1045],{},"k",[45,1047,1048,1051],{},[59,1049,1050],{},"≤",[51,1052,618],{},[45,1054,1055,1057,1059],{},[59,1056,460],{"stretchy":459},[51,1058,463],{},[59,1060,466],{"stretchy":459},[59,1062,62],{"separator":61},[625,1064,1065,1068,1074],{},[51,1066,1067],{},"v",[45,1069,1070,1072],{},[59,1071,1050],{},[51,1073,618],{},[45,1075,1076,1078,1080],{},[59,1077,460],{"stretchy":459},[51,1079,463],{},[59,1081,466],{"stretchy":459},[59,1083,466],{"stretchy":459},[59,1085,1086],{},"≜",[1088,1089,1090,1161],"mfrac",{},[45,1091,1092,1106,1109,1112,1114,1128,1145,1147],{},[1093,1094,1095,1098],"munder",{},[59,1096,1097],{},"∑",[45,1099,1100,1102,1104],{},[51,1101,296],{},[59,1103,1050],{},[51,1105,618],{},[51,1107,1108],{},"exp",[59,1110,1111],{},"⁡",[59,1113,460],{"stretchy":459},[625,1115,1116,1118,1120],{},[51,1117,1028],{},[51,1119,618],{},[45,1121,1122,1124,1126],{},[59,1123,460],{"stretchy":459},[51,1125,463],{},[59,1127,466],{"stretchy":459},[625,1129,1130,1132,1134],{},[51,1131,1045],{},[51,1133,296],{},[45,1135,1136,1138,1140,1142],{},[59,1137,460],{"stretchy":459},[51,1139,463],{},[59,1141,466],{"stretchy":459},[51,1143,1144],{"mathvariant":1002},"⊤",[59,1146,466],{"stretchy":459},[625,1148,1149,1151,1153],{},[51,1150,1067],{},[51,1152,296],{},[45,1154,1155,1157,1159],{},[59,1156,460],{"stretchy":459},[51,1158,463],{},[59,1160,466],{"stretchy":459},[45,1162,1163,1175,1177,1179,1181,1195,1211],{},[1093,1164,1165,1167],{},[59,1166,1097],{},[45,1168,1169,1171,1173],{},[51,1170,296],{},[59,1172,1050],{},[51,1174,618],{},[51,1176,1108],{},[59,1178,1111],{},[59,1180,460],{"stretchy":459},[625,1182,1183,1185,1187],{},[51,1184,1028],{},[51,1186,618],{},[45,1188,1189,1191,1193],{},[59,1190,460],{"stretchy":459},[51,1192,463],{},[59,1194,466],{"stretchy":459},[625,1196,1197,1199,1201],{},[51,1198,1045],{},[51,1200,296],{},[45,1202,1203,1205,1207,1209],{},[59,1204,460],{"stretchy":459},[51,1206,463],{},[59,1208,466],{"stretchy":459},[51,1210,1144],{"mathvariant":1002},[59,1212,466],{"stretchy":459},[89,1214,1215],{"encoding":91},"o_t^{(s)} = \\mathrm{Attention}(q_t^{(s)}, k^{(s)}_{\\le t}, v^{(s)}_{\\le t}) \\triangleq \\frac{\\sum_{i \\le t} \\exp(q_t^{(s)} k_i^{(s)\\top}) v_i^{(s)}}{\\sum_{i \\le t} \\exp(q_t^{(s)} k_i^{(s)\\top})}",[29,1217,1219,1295,1536],{"className":1218,"ariaHidden":61},[96],[29,1220,1222,1226,1286,1289,1292],{"className":1221},[100],[29,1223],{"className":1224,"style":1225},[104],"height:1.2906em;vertical-align:-0.2458em;",[29,1227,1229,1232],{"className":1228},[109],[29,1230,615],{"className":1231},[109,113],[29,1233,1235],{"className":1234},[117],[29,1236,1238,1278],{"className":1237},[121,122],[29,1239,1241,1275],{"className":1240},[126],[29,1242,1244,1255],{"className":1243,"style":771},[130],[29,1245,1246,1249],{"style":774},[29,1247],{"className":1248,"style":139},[138],[29,1250,1252],{"className":1251},[143,144,145,146],[29,1253,618],{"className":1254},[109,113,146],[29,1256,1257,1260],{"style":786},[29,1258],{"className":1259,"style":139},[138],[29,1261,1263],{"className":1262},[143,144,145,146],[29,1264,1266,1269,1272],{"className":1265},[109,146],[29,1267,460],{"className":1268},[483,146],[29,1270,463],{"className":1271},[109,113,146],[29,1273,466],{"className":1274},[490,146],[29,1276,154],{"className":1277},[153],[29,1279,1281],{"className":1280},[126],[29,1282,1284],{"className":1283,"style":814},[130],[29,1285],{},[29,1287],{"className":1288,"style":367},[171],[29,1290,508],{"className":1291},[371],[29,1293],{"className":1294,"style":367},[171],[29,1296,1298,1302,1310,1313,1375,1378,1381,1450,1453,1456,1523,1526,1529,1533],{"className":1297},[100],[29,1299],{"className":1300,"style":1301},[104],"height:1.4004em;vertical-align:-0.3556em;",[29,1303,1305],{"className":1304},[109],[29,1306,1309],{"className":1307},[109,1308],"mathrm","Attention",[29,1311,460],{"className":1312},[483],[29,1314,1316,1320],{"className":1315},[109],[29,1317,1028],{"className":1318,"style":1319},[109,113],"margin-right:0.0359em;",[29,1321,1323],{"className":1322},[117],[29,1324,1326,1367],{"className":1325},[121,122],[29,1327,1329,1364],{"className":1328},[126],[29,1330,1332,1344],{"className":1331,"style":771},[130],[29,1333,1335,1338],{"style":1334},"top:-2.4542em;margin-left:-0.0359em;margin-right:0.05em;",[29,1336],{"className":1337,"style":139},[138],[29,1339,1341],{"className":1340},[143,144,145,146],[29,1342,618],{"className":1343},[109,113,146],[29,1345,1346,1349],{"style":786},[29,1347],{"className":1348,"style":139},[138],[29,1350,1352],{"className":1351},[143,144,145,146],[29,1353,1355,1358,1361],{"className":1354},[109,146],[29,1356,460],{"className":1357},[483,146],[29,1359,463],{"className":1360},[109,113,146],[29,1362,466],{"className":1363},[490,146],[29,1365,154],{"className":1366},[153],[29,1368,1370],{"className":1369},[126],[29,1371,1373],{"className":1372,"style":814},[130],[29,1374],{},[29,1376,62],{"className":1377},[167],[29,1379],{"className":1380,"style":172},[171],[29,1382,1384,1388],{"className":1383},[109],[29,1385,1045],{"className":1386,"style":1387},[109,113],"margin-right:0.0315em;",[29,1389,1391],{"className":1390},[117],[29,1392,1394,1441],{"className":1393},[121,122],[29,1395,1397,1438],{"className":1396},[126],[29,1398,1400,1418],{"className":1399,"style":771},[130],[29,1401,1403,1406],{"style":1402},"top:-2.4396em;margin-left:-0.0315em;margin-right:0.05em;",[29,1404],{"className":1405,"style":139},[138],[29,1407,1409],{"className":1408},[143,144,145,146],[29,1410,1412,1415],{"className":1411},[109,146],[29,1413,1050],{"className":1414},[371,146],[29,1416,618],{"className":1417},[109,113,146],[29,1419,1420,1423],{"style":786},[29,1421],{"className":1422,"style":139},[138],[29,1424,1426],{"className":1425},[143,144,145,146],[29,1427,1429,1432,1435],{"className":1428},[109,146],[29,1430,460],{"className":1431},[483,146],[29,1433,463],{"className":1434},[109,113,146],[29,1436,466],{"className":1437},[490,146],[29,1439,154],{"className":1440},[153],[29,1442,1444],{"className":1443},[126],[29,1445,1448],{"className":1446,"style":1447},[130],"height:0.3556em;",[29,1449],{},[29,1451,62],{"className":1452},[167],[29,1454],{"className":1455,"style":172},[171],[29,1457,1459,1462],{"className":1458},[109],[29,1460,1067],{"className":1461,"style":1319},[109,113],[29,1463,1465],{"className":1464},[117],[29,1466,1468,1515],{"className":1467},[121,122],[29,1469,1471,1512],{"className":1470},[126],[29,1472,1474,1492],{"className":1473,"style":771},[130],[29,1475,1477,1480],{"style":1476},"top:-2.4396em;margin-left:-0.0359em;margin-right:0.05em;",[29,1478],{"className":1479,"style":139},[138],[29,1481,1483],{"className":1482},[143,144,145,146],[29,1484,1486,1489],{"className":1485},[109,146],[29,1487,1050],{"className":1488},[371,146],[29,1490,618],{"className":1491},[109,113,146],[29,1493,1494,1497],{"style":786},[29,1495],{"className":1496,"style":139},[138],[29,1498,1500],{"className":1499},[143,144,145,146],[29,1501,1503,1506,1509],{"className":1502},[109,146],[29,1504,460],{"className":1505},[483,146],[29,1507,463],{"className":1508},[109,113,146],[29,1510,466],{"className":1511},[490,146],[29,1513,154],{"className":1514},[153],[29,1516,1518],{"className":1517},[126],[29,1519,1521],{"className":1520,"style":1447},[130],[29,1522],{},[29,1524,466],{"className":1525},[490],[29,1527],{"className":1528,"style":367},[171],[29,1530,1086],{"className":1531},[371,1532],"amsrm",[29,1534],{"className":1535,"style":367},[171],[29,1537,1539,1543],{"className":1538},[100],[29,1540],{"className":1541,"style":1542},[104],"height:3.1594em;vertical-align:-1.3297em;",[29,1544,1546,1550,2043],{"className":1545},[109],[29,1547],{"className":1548},[483,1549],"nulldelimiter",[29,1551,1553],{"className":1552},[1088],[29,1554,1556,2034],{"className":1555},[121,122],[29,1557,1559,2031],{"className":1558},[126],[29,1560,1563,1766,1777],{"className":1561,"style":1562},[130],"height:1.8297em;",[29,1564,1566,1570],{"style":1565},"top:-2.11em;",[29,1567],{"className":1568,"style":1569},[138],"height:3.0448em;",[29,1571,1573,1629,1632,1635,1638,1698,1763],{"className":1572},[109],[29,1574,1577,1583],{"className":1575},[1576],"mop",[29,1578,1097],{"className":1579,"style":1582},[1576,1580,1581],"op-symbol","small-op","position:relative;top:0em;",[29,1584,1586],{"className":1585},[117],[29,1587,1589,1620],{"className":1588},[121,122],[29,1590,1592,1617],{"className":1591},[126],[29,1593,1596],{"className":1594,"style":1595},[130],"height:0.162em;",[29,1597,1599,1602],{"style":1598},"top:-2.4003em;margin-left:0em;margin-right:0.05em;",[29,1600],{"className":1601,"style":139},[138],[29,1603,1605],{"className":1604},[143,144,145,146],[29,1606,1608,1611,1614],{"className":1607},[109,146],[29,1609,296],{"className":1610},[109,113,146],[29,1612,1050],{"className":1613},[371,146],[29,1615,618],{"className":1616},[109,113,146],[29,1618,154],{"className":1619},[153],[29,1621,1623],{"className":1622},[126],[29,1624,1627],{"className":1625,"style":1626},[130],"height:0.3949em;",[29,1628],{},[29,1630],{"className":1631,"style":172},[171],[29,1633,1108],{"className":1634},[1576],[29,1636,460],{"className":1637},[483],[29,1639,1641,1644],{"className":1640},[109],[29,1642,1028],{"className":1643,"style":1319},[109,113],[29,1645,1647],{"className":1646},[117],[29,1648,1650,1690],{"className":1649},[121,122],[29,1651,1653,1687],{"className":1652},[126],[29,1654,1656,1667],{"className":1655,"style":771},[130],[29,1657,1658,1661],{"style":1334},[29,1659],{"className":1660,"style":139},[138],[29,1662,1664],{"className":1663},[143,144,145,146],[29,1665,618],{"className":1666},[109,113,146],[29,1668,1669,1672],{"style":786},[29,1670],{"className":1671,"style":139},[138],[29,1673,1675],{"className":1674},[143,144,145,146],[29,1676,1678,1681,1684],{"className":1677},[109,146],[29,1679,460],{"className":1680},[483,146],[29,1682,463],{"className":1683},[109,113,146],[29,1685,466],{"className":1686},[490,146],[29,1688,154],{"className":1689},[153],[29,1691,1693],{"className":1692},[126],[29,1694,1696],{"className":1695,"style":814},[130],[29,1697],{},[29,1699,1701,1704],{"className":1700},[109],[29,1702,1045],{"className":1703,"style":1387},[109,113],[29,1705,1707],{"className":1706},[117],[29,1708,1710,1754],{"className":1709},[121,122],[29,1711,1713,1751],{"className":1712},[126],[29,1714,1716,1728],{"className":1715,"style":771},[130],[29,1717,1719,1722],{"style":1718},"top:-2.4231em;margin-left:-0.0315em;margin-right:0.05em;",[29,1720],{"className":1721,"style":139},[138],[29,1723,1725],{"className":1724},[143,144,145,146],[29,1726,296],{"className":1727},[109,113,146],[29,1729,1730,1733],{"style":786},[29,1731],{"className":1732,"style":139},[138],[29,1734,1736],{"className":1735},[143,144,145,146],[29,1737,1739,1742,1745,1748],{"className":1738},[109,146],[29,1740,460],{"className":1741},[483,146],[29,1743,463],{"className":1744},[109,113,146],[29,1746,466],{"className":1747},[490,146],[29,1749,1144],{"className":1750},[109,146],[29,1752,154],{"className":1753},[153],[29,1755,1757],{"className":1756},[126],[29,1758,1761],{"className":1759,"style":1760},[130],"height:0.2769em;",[29,1762],{},[29,1764,466],{"className":1765},[490],[29,1767,1769,1772],{"style":1768},"top:-3.2748em;",[29,1770],{"className":1771,"style":1569},[138],[29,1773],{"className":1774,"style":1776},[1775],"frac-line","border-bottom-width:0.04em;",[29,1778,1780,1783],{"style":1779},"top:-3.8297em;",[29,1781],{"className":1782,"style":1569},[138],[29,1784,1786,1835,1838,1841,1844,1904,1967,1970],{"className":1785},[109],[29,1787,1789,1792],{"className":1788},[1576],[29,1790,1097],{"className":1791,"style":1582},[1576,1580,1581],[29,1793,1795],{"className":1794},[117],[29,1796,1798,1827],{"className":1797},[121,122],[29,1799,1801,1824],{"className":1800},[126],[29,1802,1804],{"className":1803,"style":1595},[130],[29,1805,1806,1809],{"style":1598},[29,1807],{"className":1808,"style":139},[138],[29,1810,1812],{"className":1811},[143,144,145,146],[29,1813,1815,1818,1821],{"className":1814},[109,146],[29,1816,296],{"className":1817},[109,113,146],[29,1819,1050],{"className":1820},[371,146],[29,1822,618],{"className":1823},[109,113,146],[29,1825,154],{"className":1826},[153],[29,1828,1830],{"className":1829},[126],[29,1831,1833],{"className":1832,"style":1626},[130],[29,1834],{},[29,1836],{"className":1837,"style":172},[171],[29,1839,1108],{"className":1840},[1576],[29,1842,460],{"className":1843},[483],[29,1845,1847,1850],{"className":1846},[109],[29,1848,1028],{"className":1849,"style":1319},[109,113],[29,1851,1853],{"className":1852},[117],[29,1854,1856,1896],{"className":1855},[121,122],[29,1857,1859,1893],{"className":1858},[126],[29,1860,1862,1873],{"className":1861,"style":771},[130],[29,1863,1864,1867],{"style":1334},[29,1865],{"className":1866,"style":139},[138],[29,1868,1870],{"className":1869},[143,144,145,146],[29,1871,618],{"className":1872},[109,113,146],[29,1874,1875,1878],{"style":786},[29,1876],{"className":1877,"style":139},[138],[29,1879,1881],{"className":1880},[143,144,145,146],[29,1882,1884,1887,1890],{"className":1883},[109,146],[29,1885,460],{"className":1886},[483,146],[29,1888,463],{"className":1889},[109,113,146],[29,1891,466],{"className":1892},[490,146],[29,1894,154],{"className":1895},[153],[29,1897,1899],{"className":1898},[126],[29,1900,1902],{"className":1901,"style":814},[130],[29,1903],{},[29,1905,1907,1910],{"className":1906},[109],[29,1908,1045],{"className":1909,"style":1387},[109,113],[29,1911,1913],{"className":1912},[117],[29,1914,1916,1959],{"className":1915},[121,122],[29,1917,1919,1956],{"className":1918},[126],[29,1920,1922,1933],{"className":1921,"style":771},[130],[29,1923,1924,1927],{"style":1718},[29,1925],{"className":1926,"style":139},[138],[29,1928,1930],{"className":1929},[143,144,145,146],[29,1931,296],{"className":1932},[109,113,146],[29,1934,1935,1938],{"style":786},[29,1936],{"className":1937,"style":139},[138],[29,1939,1941],{"className":1940},[143,144,145,146],[29,1942,1944,1947,1950,1953],{"className":1943},[109,146],[29,1945,460],{"className":1946},[483,146],[29,1948,463],{"className":1949},[109,113,146],[29,1951,466],{"className":1952},[490,146],[29,1954,1144],{"className":1955},[109,146],[29,1957,154],{"className":1958},[153],[29,1960,1962],{"className":1961},[126],[29,1963,1965],{"className":1964,"style":1760},[130],[29,1966],{},[29,1968,466],{"className":1969},[490],[29,1971,1973,1976],{"className":1972},[109],[29,1974,1067],{"className":1975,"style":1319},[109,113],[29,1977,1979],{"className":1978},[117],[29,1980,1982,2023],{"className":1981},[121,122],[29,1983,1985,2020],{"className":1984},[126],[29,1986,1988,2000],{"className":1987,"style":771},[130],[29,1989,1991,1994],{"style":1990},"top:-2.4231em;margin-left:-0.0359em;margin-right:0.05em;",[29,1992],{"className":1993,"style":139},[138],[29,1995,1997],{"className":1996},[143,144,145,146],[29,1998,296],{"className":1999},[109,113,146],[29,2001,2002,2005],{"style":786},[29,2003],{"className":2004,"style":139},[138],[29,2006,2008],{"className":2007},[143,144,145,146],[29,2009,2011,2014,2017],{"className":2010},[109,146],[29,2012,460],{"className":2013},[483,146],[29,2015,463],{"className":2016},[109,113,146],[29,2018,466],{"className":2019},[490,146],[29,2021,154],{"className":2022},[153],[29,2024,2026],{"className":2025},[126],[29,2027,2029],{"className":2028,"style":1760},[130],[29,2030],{},[29,2032,154],{"className":2033},[153],[29,2035,2037],{"className":2036},[126],[29,2038,2041],{"className":2039,"style":2040},[130],"height:1.3297em;",[29,2042],{},[29,2044],{"className":2045},[490,1549],[29,2047,2049],{"className":2048},[597],[29,2050,2052,2218],{"className":2051},[32],[29,2053,2055],{"className":2054},[36],[38,2056,2057],{"xmlns":40,"display":606},[42,2058,2059,2215],{},[45,2060,2061,2075,2077,2083,2098,2100,2110,2112,2115,2129,2131,2137,2151,2153,2163,2165,2167,2181,2183,2189,2203,2205],{},[625,2062,2063,2065,2067],{},[51,2064,1028],{},[51,2066,296],{},[45,2068,2069,2071,2073],{},[59,2070,460],{"stretchy":459},[51,2072,463],{},[59,2074,466],{"stretchy":459},[59,2076,508],{},[48,2078,2079,2081],{},[51,2080,53],{},[51,2082,296],{},[625,2084,2085,2088,2090],{},[51,2086,2087],{},"W",[51,2089,1028],{},[45,2091,2092,2094,2096],{},[59,2093,460],{"stretchy":459},[51,2095,463],{},[59,2097,466],{"stretchy":459},[59,2099,299],{},[301,2101,2102,2104],{},[51,2103,306],{"mathvariant":305},[48,2105,2106,2108],{},[51,2107,309],{},[51,2109,1045],{},[59,2111,62],{"separator":61},[171,2113],{"width":2114},"1em",[625,2116,2117,2119,2121],{},[51,2118,1045],{},[51,2120,296],{},[45,2122,2123,2125,2127],{},[59,2124,460],{"stretchy":459},[51,2126,463],{},[59,2128,466],{"stretchy":459},[59,2130,508],{},[48,2132,2133,2135],{},[51,2134,53],{},[51,2136,296],{},[625,2138,2139,2141,2143],{},[51,2140,2087],{},[51,2142,1045],{},[45,2144,2145,2147,2149],{},[59,2146,460],{"stretchy":459},[51,2148,463],{},[59,2150,466],{"stretchy":459},[59,2152,299],{},[301,2154,2155,2157],{},[51,2156,306],{"mathvariant":305},[48,2158,2159,2161],{},[51,2160,309],{},[51,2162,1045],{},[59,2164,62],{"separator":61},[171,2166],{"width":2114},[625,2168,2169,2171,2173],{},[51,2170,1067],{},[51,2172,296],{},[45,2174,2175,2177,2179],{},[59,2176,460],{"stretchy":459},[51,2178,463],{},[59,2180,466],{"stretchy":459},[59,2182,508],{},[48,2184,2185,2187],{},[51,2186,53],{},[51,2188,296],{},[625,2190,2191,2193,2195],{},[51,2192,2087],{},[51,2194,1067],{},[45,2196,2197,2199,2201],{},[59,2198,460],{"stretchy":459},[51,2200,463],{},[59,2202,466],{"stretchy":459},[59,2204,299],{},[301,2206,2207,2209],{},[51,2208,306],{"mathvariant":305},[48,2210,2211,2213],{},[51,2212,309],{},[51,2214,1067],{},[89,2216,2217],{"encoding":91},"q_i^{(s)} = x_i W_q^{(s)} \\in \\mathbb{R}^{d_k}, \\quad k_i^{(s)} = x_i W_k^{(s)} \\in \\mathbb{R}^{d_k}, \\quad v_i^{(s)} = x_i W_v^{(s)} \\in \\mathbb{R}^{d_v}",[29,2219,2221,2297,2418,2579,2697,2850,2967],{"className":2220,"ariaHidden":61},[96],[29,2222,2224,2228,2288,2291,2294],{"className":2223},[100],[29,2225],{"className":2226,"style":2227},[104],"height:1.3217em;vertical-align:-0.2769em;",[29,2229,2231,2234],{"className":2230},[109],[29,2232,1028],{"className":2233,"style":1319},[109,113],[29,2235,2237],{"className":2236},[117],[29,2238,2240,2280],{"className":2239},[121,122],[29,2241,2243,2277],{"className":2242},[126],[29,2244,2246,2257],{"className":2245,"style":771},[130],[29,2247,2248,2251],{"style":1990},[29,2249],{"className":2250,"style":139},[138],[29,2252,2254],{"className":2253},[143,144,145,146],[29,2255,296],{"className":2256},[109,113,146],[29,2258,2259,2262],{"style":786},[29,2260],{"className":2261,"style":139},[138],[29,2263,2265],{"className":2264},[143,144,145,146],[29,2266,2268,2271,2274],{"className":2267},[109,146],[29,2269,460],{"className":2270},[483,146],[29,2272,463],{"className":2273},[109,113,146],[29,2275,466],{"className":2276},[490,146],[29,2278,154],{"className":2279},[153],[29,2281,2283],{"className":2282},[126],[29,2284,2286],{"className":2285,"style":1760},[130],[29,2287],{},[29,2289],{"className":2290,"style":367},[171],[29,2292,508],{"className":2293},[371],[29,2295],{"className":2296,"style":367},[171],[29,2298,2300,2304,2344,2409,2412,2415],{"className":2299},[100],[29,2301],{"className":2302,"style":2303},[104],"height:1.3211em;vertical-align:-0.3831em;",[29,2305,2307,2310],{"className":2306},[109],[29,2308,53],{"className":2309},[109,113],[29,2311,2313],{"className":2312},[117],[29,2314,2316,2336],{"className":2315},[121,122],[29,2317,2319,2333],{"className":2318},[126],[29,2320,2322],{"className":2321,"style":341},[130],[29,2323,2324,2327],{"style":134},[29,2325],{"className":2326,"style":139},[138],[29,2328,2330],{"className":2329},[143,144,145,146],[29,2331,296],{"className":2332},[109,113,146],[29,2334,154],{"className":2335},[153],[29,2337,2339],{"className":2338},[126],[29,2340,2342],{"className":2341,"style":161},[130],[29,2343],{},[29,2345,2347,2351],{"className":2346},[109],[29,2348,2087],{"className":2349,"style":2350},[109,113],"margin-right:0.1389em;",[29,2352,2354],{"className":2353},[117],[29,2355,2357,2400],{"className":2356},[121,122],[29,2358,2360,2397],{"className":2359},[126],[29,2361,2364,2376],{"className":2362,"style":2363},[130],"height:0.938em;",[29,2365,2367,2370],{"style":2366},"top:-2.453em;margin-left:-0.1389em;margin-right:0.05em;",[29,2368],{"className":2369,"style":139},[138],[29,2371,2373],{"className":2372},[143,144,145,146],[29,2374,1028],{"className":2375,"style":1319},[109,113,146],[29,2377,2379,2382],{"style":2378},"top:-3.113em;margin-right:0.05em;",[29,2380],{"className":2381,"style":139},[138],[29,2383,2385],{"className":2384},[143,144,145,146],[29,2386,2388,2391,2394],{"className":2387},[109,146],[29,2389,460],{"className":2390},[483,146],[29,2392,463],{"className":2393},[109,113,146],[29,2395,466],{"className":2396},[490,146],[29,2398,154],{"className":2399},[153],[29,2401,2403],{"className":2402},[126],[29,2404,2407],{"className":2405,"style":2406},[130],"height:0.3831em;",[29,2408],{},[29,2410],{"className":2411,"style":367},[171],[29,2413,299],{"className":2414},[371],[29,2416],{"className":2417,"style":367},[171],[29,2419,2421,2424,2500,2503,2507,2510,2570,2573,2576],{"className":2420},[100],[29,2422],{"className":2423,"style":2227},[104],[29,2425,2427,2430],{"className":2426},[109],[29,2428,306],{"className":2429},[109,388],[29,2431,2433],{"className":2432},[117],[29,2434,2436],{"className":2435},[121],[29,2437,2439],{"className":2438},[126],[29,2440,2443],{"className":2441,"style":2442},[130],"height:0.8991em;",[29,2444,2445,2448],{"style":2378},[29,2446],{"className":2447,"style":139},[138],[29,2449,2451],{"className":2450},[143,144,145,146],[29,2452,2454],{"className":2453},[109,146],[29,2455,2457,2460],{"className":2456},[109,146],[29,2458,309],{"className":2459},[109,113,146],[29,2461,2463],{"className":2462},[117],[29,2464,2466,2491],{"className":2465},[121,122],[29,2467,2469,2488],{"className":2468},[126],[29,2470,2473],{"className":2471,"style":2472},[130],"height:0.3448em;",[29,2474,2476,2480],{"style":2475},"top:-2.3488em;margin-left:0em;margin-right:0.0714em;",[29,2477],{"className":2478,"style":2479},[138],"height:2.5em;",[29,2481,2485],{"className":2482},[143,2483,2484,146],"reset-size3","size1",[29,2486,1045],{"className":2487,"style":1387},[109,113,146],[29,2489,154],{"className":2490},[153],[29,2492,2494],{"className":2493},[126],[29,2495,2498],{"className":2496,"style":2497},[130],"height:0.1512em;",[29,2499],{},[29,2501,62],{"className":2502},[167],[29,2504],{"className":2505,"style":2506},[171],"margin-right:1em;",[29,2508],{"className":2509,"style":172},[171],[29,2511,2513,2516],{"className":2512},[109],[29,2514,1045],{"className":2515,"style":1387},[109,113],[29,2517,2519],{"className":2518},[117],[29,2520,2522,2562],{"className":2521},[121,122],[29,2523,2525,2559],{"className":2524},[126],[29,2526,2528,2539],{"className":2527,"style":771},[130],[29,2529,2530,2533],{"style":1718},[29,2531],{"className":2532,"style":139},[138],[29,2534,2536],{"className":2535},[143,144,145,146],[29,2537,296],{"className":2538},[109,113,146],[29,2540,2541,2544],{"style":786},[29,2542],{"className":2543,"style":139},[138],[29,2545,2547],{"className":2546},[143,144,145,146],[29,2548,2550,2553,2556],{"className":2549},[109,146],[29,2551,460],{"className":2552},[483,146],[29,2554,463],{"className":2555},[109,113,146],[29,2557,466],{"className":2558},[490,146],[29,2560,154],{"className":2561},[153],[29,2563,2565],{"className":2564},[126],[29,2566,2568],{"className":2567,"style":1760},[130],[29,2569],{},[29,2571],{"className":2572,"style":367},[171],[29,2574,508],{"className":2575},[371],[29,2577],{"className":2578,"style":367},[171],[29,2580,2582,2586,2626,2688,2691,2694],{"className":2581},[100],[29,2583],{"className":2584,"style":2585},[104],"height:1.3461em;vertical-align:-0.3013em;",[29,2587,2589,2592],{"className":2588},[109],[29,2590,53],{"className":2591},[109,113],[29,2593,2595],{"className":2594},[117],[29,2596,2598,2618],{"className":2597},[121,122],[29,2599,2601,2615],{"className":2600},[126],[29,2602,2604],{"className":2603,"style":341},[130],[29,2605,2606,2609],{"style":134},[29,2607],{"className":2608,"style":139},[138],[29,2610,2612],{"className":2611},[143,144,145,146],[29,2613,296],{"className":2614},[109,113,146],[29,2616,154],{"className":2617},[153],[29,2619,2621],{"className":2620},[126],[29,2622,2624],{"className":2623,"style":161},[130],[29,2625],{},[29,2627,2629,2632],{"className":2628},[109],[29,2630,2087],{"className":2631,"style":2350},[109,113],[29,2633,2635],{"className":2634},[117],[29,2636,2638,2679],{"className":2637},[121,122],[29,2639,2641,2676],{"className":2640},[126],[29,2642,2644,2656],{"className":2643,"style":771},[130],[29,2645,2647,2650],{"style":2646},"top:-2.3987em;margin-left:-0.1389em;margin-right:0.05em;",[29,2648],{"className":2649,"style":139},[138],[29,2651,2653],{"className":2652},[143,144,145,146],[29,2654,1045],{"className":2655,"style":1387},[109,113,146],[29,2657,2658,2661],{"style":786},[29,2659],{"className":2660,"style":139},[138],[29,2662,2664],{"className":2663},[143,144,145,146],[29,2665,2667,2670,2673],{"className":2666},[109,146],[29,2668,460],{"className":2669},[483,146],[29,2671,463],{"className":2672},[109,113,146],[29,2674,466],{"className":2675},[490,146],[29,2677,154],{"className":2678},[153],[29,2680,2682],{"className":2681},[126],[29,2683,2686],{"className":2684,"style":2685},[130],"height:0.3013em;",[29,2687],{},[29,2689],{"className":2690,"style":367},[171],[29,2692,299],{"className":2693},[371],[29,2695],{"className":2696,"style":367},[171],[29,2698,2700,2703,2772,2775,2778,2781,2841,2844,2847],{"className":2699},[100],[29,2701],{"className":2702,"style":2227},[104],[29,2704,2706,2709],{"className":2705},[109],[29,2707,306],{"className":2708},[109,388],[29,2710,2712],{"className":2711},[117],[29,2713,2715],{"className":2714},[121],[29,2716,2718],{"className":2717},[126],[29,2719,2721],{"className":2720,"style":2442},[130],[29,2722,2723,2726],{"style":2378},[29,2724],{"className":2725,"style":139},[138],[29,2727,2729],{"className":2728},[143,144,145,146],[29,2730,2732],{"className":2731},[109,146],[29,2733,2735,2738],{"className":2734},[109,146],[29,2736,309],{"className":2737},[109,113,146],[29,2739,2741],{"className":2740},[117],[29,2742,2744,2764],{"className":2743},[121,122],[29,2745,2747,2761],{"className":2746},[126],[29,2748,2750],{"className":2749,"style":2472},[130],[29,2751,2752,2755],{"style":2475},[29,2753],{"className":2754,"style":2479},[138],[29,2756,2758],{"className":2757},[143,2483,2484,146],[29,2759,1045],{"className":2760,"style":1387},[109,113,146],[29,2762,154],{"className":2763},[153],[29,2765,2767],{"className":2766},[126],[29,2768,2770],{"className":2769,"style":2497},[130],[29,2771],{},[29,2773,62],{"className":2774},[167],[29,2776],{"className":2777,"style":2506},[171],[29,2779],{"className":2780,"style":172},[171],[29,2782,2784,2787],{"className":2783},[109],[29,2785,1067],{"className":2786,"style":1319},[109,113],[29,2788,2790],{"className":2789},[117],[29,2791,2793,2833],{"className":2792},[121,122],[29,2794,2796,2830],{"className":2795},[126],[29,2797,2799,2810],{"className":2798,"style":771},[130],[29,2800,2801,2804],{"style":1990},[29,2802],{"className":2803,"style":139},[138],[29,2805,2807],{"className":2806},[143,144,145,146],[29,2808,296],{"className":2809},[109,113,146],[29,2811,2812,2815],{"style":786},[29,2813],{"className":2814,"style":139},[138],[29,2816,2818],{"className":2817},[143,144,145,146],[29,2819,2821,2824,2827],{"className":2820},[109,146],[29,2822,460],{"className":2823},[483,146],[29,2825,463],{"className":2826},[109,113,146],[29,2828,466],{"className":2829},[490,146],[29,2831,154],{"className":2832},[153],[29,2834,2836],{"className":2835},[126],[29,2837,2839],{"className":2838,"style":1760},[130],[29,2840],{},[29,2842],{"className":2843,"style":367},[171],[29,2845,508],{"className":2846},[371],[29,2848],{"className":2849,"style":367},[171],[29,2851,2853,2857,2897,2958,2961,2964],{"className":2852},[100],[29,2854],{"className":2855,"style":2856},[104],"height:1.185em;vertical-align:-0.247em;",[29,2858,2860,2863],{"className":2859},[109],[29,2861,53],{"className":2862},[109,113],[29,2864,2866],{"className":2865},[117],[29,2867,2869,2889],{"className":2868},[121,122],[29,2870,2872,2886],{"className":2871},[126],[29,2873,2875],{"className":2874,"style":341},[130],[29,2876,2877,2880],{"style":134},[29,2878],{"className":2879,"style":139},[138],[29,2881,2883],{"className":2882},[143,144,145,146],[29,2884,296],{"className":2885},[109,113,146],[29,2887,154],{"className":2888},[153],[29,2890,2892],{"className":2891},[126],[29,2893,2895],{"className":2894,"style":161},[130],[29,2896],{},[29,2898,2900,2903],{"className":2899},[109],[29,2901,2087],{"className":2902,"style":2350},[109,113],[29,2904,2906],{"className":2905},[117],[29,2907,2909,2949],{"className":2908},[121,122],[29,2910,2912,2946],{"className":2911},[126],[29,2913,2915,2926],{"className":2914,"style":2363},[130],[29,2916,2917,2920],{"style":2366},[29,2918],{"className":2919,"style":139},[138],[29,2921,2923],{"className":2922},[143,144,145,146],[29,2924,1067],{"className":2925,"style":1319},[109,113,146],[29,2927,2928,2931],{"style":2378},[29,2929],{"className":2930,"style":139},[138],[29,2932,2934],{"className":2933},[143,144,145,146],[29,2935,2937,2940,2943],{"className":2936},[109,146],[29,2938,460],{"className":2939},[483,146],[29,2941,463],{"className":2942},[109,113,146],[29,2944,466],{"className":2945},[490,146],[29,2947,154],{"className":2948},[153],[29,2950,2952],{"className":2951},[126],[29,2953,2956],{"className":2954,"style":2955},[130],"height:0.247em;",[29,2957],{},[29,2959],{"className":2960,"style":367},[171],[29,2962,299],{"className":2963},[371],[29,2965],{"className":2966,"style":367},[171],[29,2968,2970,2973],{"className":2969},[100],[29,2971],{"className":2972,"style":2442},[104],[29,2974,2976,2979],{"className":2975},[109],[29,2977,306],{"className":2978},[109,388],[29,2980,2982],{"className":2981},[117],[29,2983,2985],{"className":2984},[121],[29,2986,2988],{"className":2987},[126],[29,2989,2991],{"className":2990,"style":2442},[130],[29,2992,2993,2996],{"style":2378},[29,2994],{"className":2995,"style":139},[138],[29,2997,2999],{"className":2998},[143,144,145,146],[29,3000,3002],{"className":3001},[109,146],[29,3003,3005,3008],{"className":3004},[109,146],[29,3006,309],{"className":3007},[109,113,146],[29,3009,3011],{"className":3010},[117],[29,3012,3014,3036],{"className":3013},[121,122],[29,3015,3017,3033],{"className":3016},[126],[29,3018,3021],{"className":3019,"style":3020},[130],"height:0.1645em;",[29,3022,3024,3027],{"style":3023},"top:-2.357em;margin-left:0em;margin-right:0.0714em;",[29,3025],{"className":3026,"style":2479},[138],[29,3028,3030],{"className":3029},[143,2483,2484,146],[29,3031,1067],{"className":3032,"style":1319},[109,113,146],[29,3034,154],{"className":3035},[153],[29,3037,3039],{"className":3038},[126],[29,3040,3043],{"className":3041,"style":3042},[130],"height:0.143em;",[29,3044],{},[3046,3047,3048],"note",{},[10,3049,3050,3051,3219,3220,3276,3277,3385,3386,3494],{},"（省略了缩放因子。常见设置是 ",[29,3052,3054,3089],{"className":3053},[32],[29,3055,3057],{"className":3056},[36],[38,3058,3059],{"xmlns":40},[42,3060,3061,3086],{},[45,3062,3063,3069,3071,3077,3079,3081,3084],{},[48,3064,3065,3067],{},[51,3066,309],{},[51,3068,1045],{},[59,3070,508],{},[48,3072,3073,3075],{},[51,3074,309],{},[51,3076,1067],{},[59,3078,508],{},[51,3080,309],{},[51,3082,3083],{"mathvariant":1002},"\u002F",[51,3085,428],{},[89,3087,3088],{"encoding":91},"d_k = d_v = d\u002Fh",[29,3090,3092,3148,3204],{"className":3091,"ariaHidden":61},[96],[29,3093,3095,3099,3139,3142,3145],{"className":3094},[100],[29,3096],{"className":3097,"style":3098},[104],"height:0.8444em;vertical-align:-0.15em;",[29,3100,3102,3105],{"className":3101},[109],[29,3103,309],{"className":3104},[109,113],[29,3106,3108],{"className":3107},[117],[29,3109,3111,3131],{"className":3110},[121,122],[29,3112,3114,3128],{"className":3113},[126],[29,3115,3117],{"className":3116,"style":253},[130],[29,3118,3119,3122],{"style":134},[29,3120],{"className":3121,"style":139},[138],[29,3123,3125],{"className":3124},[143,144,145,146],[29,3126,1045],{"className":3127,"style":1387},[109,113,146],[29,3129,154],{"className":3130},[153],[29,3132,3134],{"className":3133},[126],[29,3135,3137],{"className":3136,"style":161},[130],[29,3138],{},[29,3140],{"className":3141,"style":367},[171],[29,3143,508],{"className":3144},[371],[29,3146],{"className":3147,"style":367},[171],[29,3149,3151,3154,3195,3198,3201],{"className":3150},[100],[29,3152],{"className":3153,"style":3098},[104],[29,3155,3157,3160],{"className":3156},[109],[29,3158,309],{"className":3159},[109,113],[29,3161,3163],{"className":3162},[117],[29,3164,3166,3187],{"className":3165},[121,122],[29,3167,3169,3184],{"className":3168},[126],[29,3170,3173],{"className":3171,"style":3172},[130],"height:0.1514em;",[29,3174,3175,3178],{"style":134},[29,3176],{"className":3177,"style":139},[138],[29,3179,3181],{"className":3180},[143,144,145,146],[29,3182,1067],{"className":3183,"style":1319},[109,113,146],[29,3185,154],{"className":3186},[153],[29,3188,3190],{"className":3189},[126],[29,3191,3193],{"className":3192,"style":161},[130],[29,3194],{},[29,3196],{"className":3197,"style":367},[171],[29,3199,508],{"className":3200},[371],[29,3202],{"className":3203,"style":367},[171],[29,3205,3207,3210,3213,3216],{"className":3206},[100],[29,3208],{"className":3209,"style":479},[104],[29,3211,309],{"className":3212},[109,113],[29,3214,3083],{"className":3215},[109],[29,3217,428],{"className":3218},[109,113],"。）关键在因果性。自回归生成时，新预测出来的第 ",[29,3221,3223,3242],{"className":3222},[32],[29,3224,3226],{"className":3225},[36],[38,3227,3228],{"xmlns":40},[42,3229,3230,3239],{},[45,3231,3232,3234,3237],{},[51,3233,618],{},[59,3235,3236],{},"+",[55,3238,57],{},[89,3240,3241],{"encoding":91},"t+1",[29,3243,3245,3266],{"className":3244,"ariaHidden":61},[96],[29,3246,3248,3252,3255,3259,3263],{"className":3247},[100],[29,3249],{"className":3250,"style":3251},[104],"height:0.6984em;vertical-align:-0.0833em;",[29,3253,618],{"className":3254},[109,113],[29,3256],{"className":3257,"style":3258},[171],"margin-right:0.2222em;",[29,3260,3236],{"className":3261},[3262],"mbin",[29,3264],{"className":3265,"style":3258},[171],[29,3267,3269,3273],{"className":3268},[100],[29,3270],{"className":3271,"style":3272},[104],"height:0.6444em;",[29,3274,57],{"className":3275},[109]," 个 token 不会影响已经算好的 ",[29,3278,3280,3310],{"className":3279},[32],[29,3281,3283],{"className":3282},[36],[38,3284,3285],{"xmlns":40},[42,3286,3287,3307],{},[45,3288,3289],{},[625,3290,3291,3293,3299],{},[51,3292,1045],{},[45,3294,3295,3297],{},[59,3296,1050],{},[51,3298,618],{},[45,3300,3301,3303,3305],{},[59,3302,460],{"stretchy":459},[51,3304,463],{},[59,3306,466],{"stretchy":459},[89,3308,3309],{"encoding":91},"k^{(s)}_{\\le t}",[29,3311,3313],{"className":3312,"ariaHidden":61},[96],[29,3314,3316,3319],{"className":3315},[100],[29,3317],{"className":3318,"style":1301},[104],[29,3320,3322,3325],{"className":3321},[109],[29,3323,1045],{"className":3324,"style":1387},[109,113],[29,3326,3328],{"className":3327},[117],[29,3329,3331,3377],{"className":3330},[121,122],[29,3332,3334,3374],{"className":3333},[126],[29,3335,3337,3354],{"className":3336,"style":771},[130],[29,3338,3339,3342],{"style":1402},[29,3340],{"className":3341,"style":139},[138],[29,3343,3345],{"className":3344},[143,144,145,146],[29,3346,3348,3351],{"className":3347},[109,146],[29,3349,1050],{"className":3350},[371,146],[29,3352,618],{"className":3353},[109,113,146],[29,3355,3356,3359],{"style":786},[29,3357],{"className":3358,"style":139},[138],[29,3360,3362],{"className":3361},[143,144,145,146],[29,3363,3365,3368,3371],{"className":3364},[109,146],[29,3366,460],{"className":3367},[483,146],[29,3369,463],{"className":3370},[109,113,146],[29,3372,466],{"className":3373},[490,146],[29,3375,154],{"className":3376},[153],[29,3378,3380],{"className":3379},[126],[29,3381,3383],{"className":3382,"style":1447},[130],[29,3384],{}," 和 ",[29,3387,3389,3419],{"className":3388},[32],[29,3390,3392],{"className":3391},[36],[38,3393,3394],{"xmlns":40},[42,3395,3396,3416],{},[45,3397,3398],{},[625,3399,3400,3402,3408],{},[51,3401,1067],{},[45,3403,3404,3406],{},[59,3405,1050],{},[51,3407,618],{},[45,3409,3410,3412,3414],{},[59,3411,460],{"stretchy":459},[51,3413,463],{},[59,3415,466],{"stretchy":459},[89,3417,3418],{"encoding":91},"v^{(s)}_{\\le t}",[29,3420,3422],{"className":3421,"ariaHidden":61},[96],[29,3423,3425,3428],{"className":3424},[100],[29,3426],{"className":3427,"style":1301},[104],[29,3429,3431,3434],{"className":3430},[109],[29,3432,1067],{"className":3433,"style":1319},[109,113],[29,3435,3437],{"className":3436},[117],[29,3438,3440,3486],{"className":3439},[121,122],[29,3441,3443,3483],{"className":3442},[126],[29,3444,3446,3463],{"className":3445,"style":771},[130],[29,3447,3448,3451],{"style":1476},[29,3449],{"className":3450,"style":139},[138],[29,3452,3454],{"className":3453},[143,144,145,146],[29,3455,3457,3460],{"className":3456},[109,146],[29,3458,1050],{"className":3459},[371,146],[29,3461,618],{"className":3462},[109,113,146],[29,3464,3465,3468],{"style":786},[29,3466],{"className":3467,"style":139},[138],[29,3469,3471],{"className":3470},[143,144,145,146],[29,3472,3474,3477,3480],{"className":3473},[109,146],[29,3475,460],{"className":3476},[483,146],[29,3478,463],{"className":3479},[109,113,146],[29,3481,466],{"className":3482},[490,146],[29,3484,154],{"className":3485},[153],[29,3487,3489],{"className":3488},[126],[29,3490,3492],{"className":3491,"style":1447},[130],[29,3493],{},"，所以这部分可以缓存下来，避免重复计算。这就是 KV Cache。它的大小约等于：",[29,3496,3498],{"className":3497},[597],[29,3499,3501,3543],{"className":3500},[32],[29,3502,3504],{"className":3503},[36],[38,3505,3506],{"xmlns":40,"display":606},[42,3507,3508,3540],{},[45,3509,3510,3512,3515,3518,3520,3522,3524,3530,3532,3535,3537],{},[55,3511,69],{},[59,3513,3514],{},"×",[51,3516,3517],{},"L",[59,3519,3514],{},[51,3521,428],{},[59,3523,3514],{},[48,3525,3526,3528],{},[51,3527,309],{},[51,3529,1045],{},[59,3531,3514],{},[51,3533,3534],{},"N",[59,3536,3514],{},[76,3538,3539],{},"bytes",[89,3541,3542],{"encoding":91},"2 \\times L \\times h \\times d_k \\times N \\times \\text{bytes}",[29,3544,3546,3565,3584,3603,3658,3677],{"className":3545,"ariaHidden":61},[96],[29,3547,3549,3553,3556,3559,3562],{"className":3548},[100],[29,3550],{"className":3551,"style":3552},[104],"height:0.7278em;vertical-align:-0.0833em;",[29,3554,69],{"className":3555},[109],[29,3557],{"className":3558,"style":3258},[171],[29,3560,3514],{"className":3561},[3262],[29,3563],{"className":3564,"style":3258},[171],[29,3566,3568,3572,3575,3578,3581],{"className":3567},[100],[29,3569],{"className":3570,"style":3571},[104],"height:0.7667em;vertical-align:-0.0833em;",[29,3573,3517],{"className":3574},[109,113],[29,3576],{"className":3577,"style":3258},[171],[29,3579,3514],{"className":3580},[3262],[29,3582],{"className":3583,"style":3258},[171],[29,3585,3587,3591,3594,3597,3600],{"className":3586},[100],[29,3588],{"className":3589,"style":3590},[104],"height:0.7778em;vertical-align:-0.0833em;",[29,3592,428],{"className":3593},[109,113],[29,3595],{"className":3596,"style":3258},[171],[29,3598,3514],{"className":3599},[3262],[29,3601],{"className":3602,"style":3258},[171],[29,3604,3606,3609,3649,3652,3655],{"className":3605},[100],[29,3607],{"className":3608,"style":3098},[104],[29,3610,3612,3615],{"className":3611},[109],[29,3613,309],{"className":3614},[109,113],[29,3616,3618],{"className":3617},[117],[29,3619,3621,3641],{"className":3620},[121,122],[29,3622,3624,3638],{"className":3623},[126],[29,3625,3627],{"className":3626,"style":253},[130],[29,3628,3629,3632],{"style":134},[29,3630],{"className":3631,"style":139},[138],[29,3633,3635],{"className":3634},[143,144,145,146],[29,3636,1045],{"className":3637,"style":1387},[109,113,146],[29,3639,154],{"className":3640},[153],[29,3642,3644],{"className":3643},[126],[29,3645,3647],{"className":3646,"style":161},[130],[29,3648],{},[29,3650],{"className":3651,"style":3258},[171],[29,3653,3514],{"className":3654},[3262],[29,3656],{"className":3657,"style":3258},[171],[29,3659,3661,3664,3668,3671,3674],{"className":3660},[100],[29,3662],{"className":3663,"style":3571},[104],[29,3665,3534],{"className":3666,"style":3667},[109,113],"margin-right:0.109em;",[29,3669],{"className":3670,"style":3258},[171],[29,3672,3514],{"className":3673},[3262],[29,3675],{"className":3676,"style":3258},[171],[29,3678,3680,3683],{"className":3679},[100],[29,3681],{"className":3682,"style":556},[104],[29,3684,3687],{"className":3685},[109,3686],"text",[29,3688,3539],{"className":3689},[109],[10,3691,3692,3693,3722,3723,3751,3752,3780,3781,3809],{},"其中 ",[29,3694,3696,3709],{"className":3695},[32],[29,3697,3699],{"className":3698},[36],[38,3700,3701],{"xmlns":40},[42,3702,3703,3707],{},[45,3704,3705],{},[51,3706,3517],{},[89,3708,3517],{"encoding":91},[29,3710,3712],{"className":3711,"ariaHidden":61},[96],[29,3713,3715,3719],{"className":3714},[100],[29,3716],{"className":3717,"style":3718},[104],"height:0.6833em;",[29,3720,3517],{"className":3721},[109,113]," 是层数，",[29,3724,3726,3739],{"className":3725},[32],[29,3727,3729],{"className":3728},[36],[38,3730,3731],{"xmlns":40},[42,3732,3733,3737],{},[45,3734,3735],{},[51,3736,3534],{},[89,3738,3534],{"encoding":91},[29,3740,3742],{"className":3741,"ariaHidden":61},[96],[29,3743,3745,3748],{"className":3744},[100],[29,3746],{"className":3747,"style":3718},[104],[29,3749,3534],{"className":3750,"style":3667},[109,113]," 是序列长度。注意这里面的 ",[29,3753,3755,3768],{"className":3754},[32],[29,3756,3758],{"className":3757},[36],[38,3759,3760],{"xmlns":40},[42,3761,3762,3766],{},[45,3763,3764],{},[51,3765,3534],{},[89,3767,3534],{"encoding":91},[29,3769,3771],{"className":3770,"ariaHidden":61},[96],[29,3772,3774,3777],{"className":3773},[100],[29,3775],{"className":3776,"style":3718},[104],[29,3778,3534],{"className":3779,"style":3667},[109,113],"。模型参数是个常数，选定模型就不再变了；但 KV Cache 随 ",[29,3782,3784,3797],{"className":3783},[32],[29,3785,3787],{"className":3786},[36],[38,3788,3789],{"xmlns":40},[42,3790,3791,3795],{},[45,3792,3793],{},[51,3794,3534],{},[89,3796,3534],{"encoding":91},[29,3798,3800],{"className":3799,"ariaHidden":61},[96],[29,3801,3803,3806],{"className":3802},[100],[29,3804],{"className":3805,"style":3718},[104],[29,3807,3534],{"className":3808,"style":3667},[109,113]," 线性增长。Context 足够长的时候，它会变成显存里的主导项，可能超出一张卡甚至一台机（8 张卡）的总显存。举个数。按 GPT-3 的配置算：96 层，96 个 attention head，每个 head 维度 128。单个 token 的 KV Cache 是",[29,3811,3813],{"className":3812},[597],[29,3814,3816,3853],{"className":3815},[32],[29,3817,3819],{"className":3818},[36],[38,3820,3821],{"xmlns":40,"display":606},[42,3822,3823,3850],{},[45,3824,3825,3827,3829,3832,3834,3836,3838,3841,3844,3847],{},[55,3826,69],{},[59,3828,3514],{},[55,3830,3831],{},"96",[59,3833,3514],{},[55,3835,3831],{},[59,3837,3514],{},[55,3839,3840],{},"128",[59,3842,3843],{},"≈",[55,3845,3846],{},"2.36",[76,3848,3849],{},"M",[89,3851,3852],{"encoding":91},"2 \\times 96 \\times 96 \\times 128 \\approx 2.36\\text{M}",[29,3854,3856,3874,3892,3910,3928],{"className":3855,"ariaHidden":61},[96],[29,3857,3859,3862,3865,3868,3871],{"className":3858},[100],[29,3860],{"className":3861,"style":3552},[104],[29,3863,69],{"className":3864},[109],[29,3866],{"className":3867,"style":3258},[171],[29,3869,3514],{"className":3870},[3262],[29,3872],{"className":3873,"style":3258},[171],[29,3875,3877,3880,3883,3886,3889],{"className":3876},[100],[29,3878],{"className":3879,"style":3552},[104],[29,3881,3831],{"className":3882},[109],[29,3884],{"className":3885,"style":3258},[171],[29,3887,3514],{"className":3888},[3262],[29,3890],{"className":3891,"style":3258},[171],[29,3893,3895,3898,3901,3904,3907],{"className":3894},[100],[29,3896],{"className":3897,"style":3552},[104],[29,3899,3831],{"className":3900},[109],[29,3902],{"className":3903,"style":3258},[171],[29,3905,3514],{"className":3906},[3262],[29,3908],{"className":3909,"style":3258},[171],[29,3911,3913,3916,3919,3922,3925],{"className":3912},[100],[29,3914],{"className":3915,"style":3272},[104],[29,3917,3840],{"className":3918},[109],[29,3920],{"className":3921,"style":367},[171],[29,3923,3843],{"className":3924},[371],[29,3926],{"className":3927,"style":367},[171],[29,3929,3931,3934,3937],{"className":3930},[100],[29,3932],{"className":3933,"style":3718},[104],[29,3935,3846],{"className":3936},[109],[29,3938,3940],{"className":3939},[109,3686],[29,3941,3849],{"className":3942},[109],[10,3944,3945],{},"个参数。按 2 字节精度存，就是 4.7 MB。一个 token 4.7 MB。128k context 就是 600 GB 量级。这个数字基本说明了为什么长上下文不是\"再多加个位置编码\"就能解决的。它是个显存和带宽问题。而 GPU 部署有个很不讲道理的原则：能一张卡放下的，就不要跨多张卡；能一台机放下的，就不要跨多台机。因为卡内通信带宽 > 卡间通信带宽 > 机间通信带宽。木桶效应，跨的设备越多，被最慢那段拖累得越狠。",[10,3947,3948],{},"单卡 H100 内 SRAM 到 HBM 的带宽已经有 3 TB\u002Fs，对 short context 来说这依然是瓶颈，更不用说更慢的卡间和机间了。",[21,3950,3952],{"id":3951},"prefill-与-decode","Prefill 与 Decode",[10,3954,3955,3956,3959,3960,413,3963,3966],{},"LLM 推理分两个阶段，它们的成本结构不同。",[14,3957,3958],{},"Prefill","：把输入的所有 token 并行算一遍，同时把 KV Cache 建起来。所有 token 一起算，矩阵运算排得很满，算力基本能吃满。它是 ",[14,3961,3962],{},"compute bound",[14,3964,3965],{},"Decode","：之后逐个生成 token。这里有个容易漏掉的点。",[10,3968,3969,3970,3973],{},"每生成一个 token，计算量很小，但要读的东西一点都不少：",[14,3971,3972],{},"模型权重","要整体过一遍，KV Cache 也要读一遍。",[10,3975,3976,3977,3980],{},"不是读一部分权重，也不是只读 attention 那几层。每一步都要把参与计算的权重从 HBM 搬进 SM。算得少，搬得多。所以 decode 是 ",[14,3978,3979],{},"memory bound","。同一个模型，两个阶段的瓶颈完全不同。如果一个架构在两种形式下数学等价，那就可以 prefill 用一种算，decode 用另一种算，各取所需。MLA 就是这么干的。",[10,3982,3983],{},"后面 V4.1 那一节会把这两个瓶颈各自的账算一遍。最后把账单写全：",[29,3985,3987],{"className":3986},[597],[29,3988,3990,4037],{"className":3989},[32],[29,3991,3993],{"className":3992},[36],[38,3994,3995],{"xmlns":40,"display":606},[42,3996,3997,4034],{},[45,3998,3999,4007,4009,4016,4018,4025,4027],{},[48,4000,4001,4004],{},[51,4002,4003],{},"C",[76,4005,4006],{},"total",[59,4008,508],{},[48,4010,4011,4013],{},[51,4012,4003],{},[76,4014,4015],{},"attention",[59,4017,3236],{},[48,4019,4020,4022],{},[51,4021,4003],{},[76,4023,4024],{},"FFN\u002FMoE",[59,4026,3236],{},[48,4028,4029,4031],{},[51,4030,4003],{},[76,4032,4033],{},"memory movement",[89,4035,4036],{"encoding":91},"C_{\\text{total}} = C_{\\text{attention}} + C_{\\text{FFN\u002FMoE}} + C_{\\text{memory movement}}",[29,4038,4040,4104,4166,4230],{"className":4039,"ariaHidden":61},[96],[29,4041,4043,4047,4095,4098,4101],{"className":4042},[100],[29,4044],{"className":4045,"style":4046},[104],"height:0.8333em;vertical-align:-0.15em;",[29,4048,4050,4054],{"className":4049},[109],[29,4051,4003],{"className":4052,"style":4053},[109,113],"margin-right:0.0715em;",[29,4055,4057],{"className":4056},[117],[29,4058,4060,4087],{"className":4059},[121,122],[29,4061,4063,4084],{"className":4062},[126],[29,4064,4066],{"className":4065,"style":253},[130],[29,4067,4069,4072],{"style":4068},"top:-2.55em;margin-left:-0.0715em;margin-right:0.05em;",[29,4070],{"className":4071,"style":139},[138],[29,4073,4075],{"className":4074},[143,144,145,146],[29,4076,4078],{"className":4077},[109,146],[29,4079,4081],{"className":4080},[109,3686,146],[29,4082,4006],{"className":4083},[109,146],[29,4085,154],{"className":4086},[153],[29,4088,4090],{"className":4089},[126],[29,4091,4093],{"className":4092,"style":161},[130],[29,4094],{},[29,4096],{"className":4097,"style":367},[171],[29,4099,508],{"className":4100},[371],[29,4102],{"className":4103,"style":367},[171],[29,4105,4107,4110,4157,4160,4163],{"className":4106},[100],[29,4108],{"className":4109,"style":4046},[104],[29,4111,4113,4116],{"className":4112},[109],[29,4114,4003],{"className":4115,"style":4053},[109,113],[29,4117,4119],{"className":4118},[117],[29,4120,4122,4149],{"className":4121},[121,122],[29,4123,4125,4146],{"className":4124},[126],[29,4126,4129],{"className":4127,"style":4128},[130],"height:0.3175em;",[29,4130,4131,4134],{"style":4068},[29,4132],{"className":4133,"style":139},[138],[29,4135,4137],{"className":4136},[143,144,145,146],[29,4138,4140],{"className":4139},[109,146],[29,4141,4143],{"className":4142},[109,3686,146],[29,4144,4015],{"className":4145},[109,146],[29,4147,154],{"className":4148},[153],[29,4150,4152],{"className":4151},[126],[29,4153,4155],{"className":4154,"style":161},[130],[29,4156],{},[29,4158],{"className":4159,"style":3258},[171],[29,4161,3236],{"className":4162},[3262],[29,4164],{"className":4165,"style":3258},[171],[29,4167,4169,4173,4221,4224,4227],{"className":4168},[100],[29,4170],{"className":4171,"style":4172},[104],"height:1.0385em;vertical-align:-0.3552em;",[29,4174,4176,4179],{"className":4175},[109],[29,4177,4003],{"className":4178,"style":4053},[109,113],[29,4180,4182],{"className":4181},[117],[29,4183,4185,4212],{"className":4184},[121,122],[29,4186,4188,4209],{"className":4187},[126],[29,4189,4191],{"className":4190,"style":2472},[130],[29,4192,4194,4197],{"style":4193},"top:-2.5198em;margin-left:-0.0715em;margin-right:0.05em;",[29,4195],{"className":4196,"style":139},[138],[29,4198,4200],{"className":4199},[143,144,145,146],[29,4201,4203],{"className":4202},[109,146],[29,4204,4206],{"className":4205},[109,3686,146],[29,4207,4024],{"className":4208},[109,146],[29,4210,154],{"className":4211},[153],[29,4213,4215],{"className":4214},[126],[29,4216,4219],{"className":4217,"style":4218},[130],"height:0.3552em;",[29,4220],{},[29,4222],{"className":4223,"style":3258},[171],[29,4225,3236],{"className":4226},[3262],[29,4228],{"className":4229,"style":3258},[171],[29,4231,4233,4237],{"className":4232},[100],[29,4234],{"className":4235,"style":4236},[104],"height:0.9694em;vertical-align:-0.2861em;",[29,4238,4240,4243],{"className":4239},[109],[29,4241,4003],{"className":4242,"style":4053},[109,113],[29,4244,4246],{"className":4245},[117],[29,4247,4249,4275],{"className":4248},[121,122],[29,4250,4252,4272],{"className":4251},[126],[29,4253,4255],{"className":4254,"style":711},[130],[29,4256,4257,4260],{"style":4068},[29,4258],{"className":4259,"style":139},[138],[29,4261,4263],{"className":4262},[143,144,145,146],[29,4264,4266],{"className":4265},[109,146],[29,4267,4269],{"className":4268},[109,3686,146],[29,4270,4033],{"className":4271},[109,146],[29,4273,154],{"className":4274},[153],[29,4276,4278],{"className":4277},[126],[29,4279,4282],{"className":4280,"style":4281},[130],"height:0.2861em;",[29,4283],{},[10,4285,4286,4287,4363,4364,4392,4393,4421,4422,4505,4506,4585,4586,4614,4615,4679,4680,4744,4745,4773],{},"账单上还有另外两项，而且它们经常被无视。第二项 ",[29,4288,4290,4308],{"className":4289},[32],[29,4291,4293],{"className":4292},[36],[38,4294,4295],{"xmlns":40},[42,4296,4297,4305],{},[45,4298,4299],{},[48,4300,4301,4303],{},[51,4302,4003],{},[76,4304,4024],{},[89,4306,4307],{"encoding":91},"C_{\\text{FFN\u002FMoE}}",[29,4309,4311],{"className":4310,"ariaHidden":61},[96],[29,4312,4314,4317],{"className":4313},[100],[29,4315],{"className":4316,"style":4172},[104],[29,4318,4320,4323],{"className":4319},[109],[29,4321,4003],{"className":4322,"style":4053},[109,113],[29,4324,4326],{"className":4325},[117],[29,4327,4329,4355],{"className":4328},[121,122],[29,4330,4332,4352],{"className":4331},[126],[29,4333,4335],{"className":4334,"style":2472},[130],[29,4336,4337,4340],{"style":4193},[29,4338],{"className":4339,"style":139},[138],[29,4341,4343],{"className":4342},[143,144,145,146],[29,4344,4346],{"className":4345},[109,146],[29,4347,4349],{"className":4348},[109,3686,146],[29,4350,4024],{"className":4351},[109,146],[29,4353,154],{"className":4354},[153],[29,4356,4358],{"className":4357},[126],[29,4359,4361],{"className":4360,"style":4218},[130],[29,4362],{}," 是前馈层的计算。MoE（Mixture of Experts）就是把这层换成多个专家网络，每个 token 只激活其中少数几个。它跟序列长度 ",[29,4365,4367,4380],{"className":4366},[32],[29,4368,4370],{"className":4369},[36],[38,4371,4372],{"xmlns":40},[42,4373,4374,4378],{},[45,4375,4376],{},[51,4377,3534],{},[89,4379,3534],{"encoding":91},[29,4381,4383],{"className":4382,"ariaHidden":61},[96],[29,4384,4386,4389],{"className":4385},[100],[29,4387],{"className":4388,"style":3718},[104],[29,4390,3534],{"className":4391,"style":3667},[109,113]," 成正比，跟模型宽度 ",[29,4394,4396,4409],{"className":4395},[32],[29,4397,4399],{"className":4398},[36],[38,4400,4401],{"xmlns":40},[42,4402,4403,4407],{},[45,4404,4405],{},[51,4406,309],{},[89,4408,309],{"encoding":91},[29,4410,4412],{"className":4411,"ariaHidden":61},[96],[29,4413,4415,4418],{"className":4414},[100],[29,4416],{"className":4417,"style":440},[104],[29,4419,309],{"className":4420},[109,113]," 的平方成正比，所以量级是 ",[29,4423,4425,4452],{"className":4424},[32],[29,4426,4428],{"className":4427},[36],[38,4429,4430],{"xmlns":40},[42,4431,4432,4449],{},[45,4433,4434,4437,4439,4441,4447],{},[51,4435,4436],{},"O",[59,4438,460],{"stretchy":459},[51,4440,3534],{},[301,4442,4443,4445],{},[51,4444,309],{},[55,4446,69],{},[59,4448,466],{"stretchy":459},[89,4450,4451],{"encoding":91},"O(Nd^2)",[29,4453,4455],{"className":4454,"ariaHidden":61},[96],[29,4456,4458,4462,4466,4469,4472,4502],{"className":4457},[100],[29,4459],{"className":4460,"style":4461},[104],"height:1.0641em;vertical-align:-0.25em;",[29,4463,4436],{"className":4464,"style":4465},[109,113],"margin-right:0.0278em;",[29,4467,460],{"className":4468},[483],[29,4470,3534],{"className":4471,"style":3667},[109,113],[29,4473,4475,4478],{"className":4474},[109],[29,4476,309],{"className":4477},[109,113],[29,4479,4481],{"className":4480},[117],[29,4482,4484],{"className":4483},[121],[29,4485,4487],{"className":4486},[126],[29,4488,4491],{"className":4489,"style":4490},[130],"height:0.8141em;",[29,4492,4493,4496],{"style":403},[29,4494],{"className":4495,"style":139},[138],[29,4497,4499],{"className":4498},[143,144,145,146],[29,4500,69],{"className":4501},[109,146],[29,4503,466],{"className":4504},[490],"。attention 那项是 ",[29,4507,4509,4535],{"className":4508},[32],[29,4510,4512],{"className":4511},[36],[38,4513,4514],{"xmlns":40},[42,4515,4516,4532],{},[45,4517,4518,4520,4522,4528,4530],{},[51,4519,4436],{},[59,4521,460],{"stretchy":459},[301,4523,4524,4526],{},[51,4525,3534],{},[55,4527,69],{},[51,4529,309],{},[59,4531,466],{"stretchy":459},[89,4533,4534],{"encoding":91},"O(N^2 d)",[29,4536,4538],{"className":4537,"ariaHidden":61},[96],[29,4539,4541,4544,4547,4550,4579,4582],{"className":4540},[100],[29,4542],{"className":4543,"style":4461},[104],[29,4545,4436],{"className":4546,"style":4465},[109,113],[29,4548,460],{"className":4549},[483],[29,4551,4553,4556],{"className":4552},[109],[29,4554,3534],{"className":4555,"style":3667},[109,113],[29,4557,4559],{"className":4558},[117],[29,4560,4562],{"className":4561},[121],[29,4563,4565],{"className":4564},[126],[29,4566,4568],{"className":4567,"style":4490},[130],[29,4569,4570,4573],{"style":403},[29,4571],{"className":4572,"style":139},[138],[29,4574,4576],{"className":4575},[143,144,145,146],[29,4577,69],{"className":4578},[109,146],[29,4580,309],{"className":4581},[109,113],[29,4583,466],{"className":4584},[490],"。注意这两个量级里的指数位置不一样：",[29,4587,4589,4602],{"className":4588},[32],[29,4590,4592],{"className":4591},[36],[38,4593,4594],{"xmlns":40},[42,4595,4596,4600],{},[45,4597,4598],{},[51,4599,3534],{},[89,4601,3534],{"encoding":91},[29,4603,4605],{"className":4604,"ariaHidden":61},[96],[29,4606,4608,4611],{"className":4607},[100],[29,4609],{"className":4610,"style":3718},[104],[29,4612,3534],{"className":4613,"style":3667},[109,113]," 小的时候，",[29,4616,4618,4638],{"className":4617},[32],[29,4619,4621],{"className":4620},[36],[38,4622,4623],{"xmlns":40},[42,4624,4625,4635],{},[45,4626,4627,4633],{},[301,4628,4629,4631],{},[51,4630,3534],{},[55,4632,69],{},[51,4634,309],{},[89,4636,4637],{"encoding":91},"N^2 d",[29,4639,4641],{"className":4640,"ariaHidden":61},[96],[29,4642,4644,4647,4676],{"className":4643},[100],[29,4645],{"className":4646,"style":4490},[104],[29,4648,4650,4653],{"className":4649},[109],[29,4651,3534],{"className":4652,"style":3667},[109,113],[29,4654,4656],{"className":4655},[117],[29,4657,4659],{"className":4658},[121],[29,4660,4662],{"className":4661},[126],[29,4663,4665],{"className":4664,"style":4490},[130],[29,4666,4667,4670],{"style":403},[29,4668],{"className":4669,"style":139},[138],[29,4671,4673],{"className":4672},[143,144,145,146],[29,4674,69],{"className":4675},[109,146],[29,4677,309],{"className":4678},[109,113]," 打不过 ",[29,4681,4683,4703],{"className":4682},[32],[29,4684,4686],{"className":4685},[36],[38,4687,4688],{"xmlns":40},[42,4689,4690,4700],{},[45,4691,4692,4694],{},[51,4693,3534],{},[301,4695,4696,4698],{},[51,4697,309],{},[55,4699,69],{},[89,4701,4702],{"encoding":91},"Nd^2",[29,4704,4706],{"className":4705,"ariaHidden":61},[96],[29,4707,4709,4712,4715],{"className":4708},[100],[29,4710],{"className":4711,"style":4490},[104],[29,4713,3534],{"className":4714,"style":3667},[109,113],[29,4716,4718,4721],{"className":4717},[109],[29,4719,309],{"className":4720},[109,113],[29,4722,4724],{"className":4723},[117],[29,4725,4727],{"className":4726},[121],[29,4728,4730],{"className":4729},[126],[29,4731,4733],{"className":4732,"style":4490},[130],[29,4734,4735,4738],{"style":403},[29,4736],{"className":4737,"style":139},[138],[29,4739,4741],{"className":4740},[143,144,145,146],[29,4742,69],{"className":4743},[109,146],"；",[29,4746,4748,4761],{"className":4747},[32],[29,4749,4751],{"className":4750},[36],[38,4752,4753],{"xmlns":40},[42,4754,4755,4759],{},[45,4756,4757],{},[51,4758,3534],{},[89,4760,3534],{"encoding":91},[29,4762,4764],{"className":4763,"ariaHidden":61},[96],[29,4765,4767,4770],{"className":4766},[100],[29,4768],{"className":4769,"style":3718},[104],[29,4771,3534],{"className":4772,"style":3667},[109,113]," 大起来之后才反过来。",[10,4775,4776,4777,4805,4806,4865,4866,4944,4945,5021],{},"这也是为什么在短 context 时代大家理所当然地只盯 attention。那时候它确实是瓶颈。但当 ",[29,4778,4780,4793],{"className":4779},[32],[29,4781,4783],{"className":4782},[36],[38,4784,4785],{"xmlns":40},[42,4786,4787,4791],{},[45,4788,4789],{},[51,4790,3534],{},[89,4792,3534],{"encoding":91},[29,4794,4796],{"className":4795,"ariaHidden":61},[96],[29,4797,4799,4802],{"className":4798},[100],[29,4800],{"className":4801,"style":3718},[104],[29,4803,3534],{"className":4804,"style":3667},[109,113]," 从 4k 涨到 1M，情况变了；再等到 sparse attention 把 ",[29,4807,4809,4827],{"className":4808},[32],[29,4810,4812],{"className":4811},[36],[38,4813,4814],{"xmlns":40},[42,4815,4816,4824],{},[45,4817,4818],{},[301,4819,4820,4822],{},[51,4821,3534],{},[55,4823,69],{},[89,4825,4826],{"encoding":91},"N^2",[29,4828,4830],{"className":4829,"ariaHidden":61},[96],[29,4831,4833,4836],{"className":4832},[100],[29,4834],{"className":4835,"style":4490},[104],[29,4837,4839,4842],{"className":4838},[109],[29,4840,3534],{"className":4841,"style":3667},[109,113],[29,4843,4845],{"className":4844},[117],[29,4846,4848],{"className":4847},[121],[29,4849,4851],{"className":4850},[126],[29,4852,4854],{"className":4853,"style":4490},[130],[29,4855,4856,4859],{"style":403},[29,4857],{"className":4858,"style":139},[138],[29,4860,4862],{"className":4861},[143,144,145,146],[29,4863,69],{"className":4864},[109,146]," 压下去，",[29,4867,4869,4894],{"className":4868},[32],[29,4870,4872],{"className":4871},[36],[38,4873,4874],{"xmlns":40},[42,4875,4876,4892],{},[45,4877,4878,4880,4882,4884,4890],{},[51,4879,4436],{},[59,4881,460],{"stretchy":459},[51,4883,3534],{},[301,4885,4886,4888],{},[51,4887,309],{},[55,4889,69],{},[59,4891,466],{"stretchy":459},[89,4893,4451],{"encoding":91},[29,4895,4897],{"className":4896,"ariaHidden":61},[96],[29,4898,4900,4903,4906,4909,4912,4941],{"className":4899},[100],[29,4901],{"className":4902,"style":4461},[104],[29,4904,4436],{"className":4905,"style":4465},[109,113],[29,4907,460],{"className":4908},[483],[29,4910,3534],{"className":4911,"style":3667},[109,113],[29,4913,4915,4918],{"className":4914},[109],[29,4916,309],{"className":4917},[109,113],[29,4919,4921],{"className":4920},[117],[29,4922,4924],{"className":4923},[121],[29,4925,4927],{"className":4926},[126],[29,4928,4930],{"className":4929,"style":4490},[130],[29,4931,4932,4935],{"style":403},[29,4933],{"className":4934,"style":139},[138],[29,4936,4938],{"className":4937},[143,144,145,146],[29,4939,69],{"className":4940},[109,146],[29,4942,466],{"className":4943},[490]," 这一项就浮到水面上了。第三项 ",[29,4946,4948,4966],{"className":4947},[32],[29,4949,4951],{"className":4950},[36],[38,4952,4953],{"xmlns":40},[42,4954,4955,4963],{},[45,4956,4957],{},[48,4958,4959,4961],{},[51,4960,4003],{},[76,4962,4033],{},[89,4964,4965],{"encoding":91},"C_{\\text{memory movement}}",[29,4967,4969],{"className":4968,"ariaHidden":61},[96],[29,4970,4972,4975],{"className":4971},[100],[29,4973],{"className":4974,"style":4236},[104],[29,4976,4978,4981],{"className":4977},[109],[29,4979,4003],{"className":4980,"style":4053},[109,113],[29,4982,4984],{"className":4983},[117],[29,4985,4987,5013],{"className":4986},[121,122],[29,4988,4990,5010],{"className":4989},[126],[29,4991,4993],{"className":4992,"style":711},[130],[29,4994,4995,4998],{"style":4068},[29,4996],{"className":4997,"style":139},[138],[29,4999,5001],{"className":5000},[143,144,145,146],[29,5002,5004],{"className":5003},[109,146],[29,5005,5007],{"className":5006},[109,3686,146],[29,5008,4033],{"className":5009},[109,146],[29,5011,154],{"className":5012},[153],[29,5014,5016],{"className":5015},[126],[29,5017,5019],{"className":5018,"style":4281},[130],[29,5020],{}," 是最容易漏的。GPU 上算一个 FLOP 很快，搬一次数据慢得多，所以\"减少 KV Cache\"的收益很多时候落在省搬运，而不在省显存。但这件事有个限度：decode 每步要读的权重是固定的，搬不动，KV Cache 压到比它更小之后，继续压的边际收益就下降。这条限度解释了后面很多设计的取舍，也解释了为什么 1M context 之后，战场转向每层的计算和权重：账单上还能砍的地方换位置了。",[21,5023,5025],{"id":5024},"mqa-与-gqa","MQA 与 GQA",[10,5027,5028,5029,5071,5072,5110,5111,5148],{},"既然 KV Cache 是问题，最朴素的想法就是：少存点。MQA（Multi-Query Attention）直接让所有 attention head 共享同一份 K、V。用公式说，就是把所有 ",[29,5030,5032,5050],{"className":5031},[32],[29,5033,5035],{"className":5034},[36],[38,5036,5037],{"xmlns":40},[42,5038,5039,5047],{},[45,5040,5041,5043,5045],{},[51,5042,1045],{},[59,5044,62],{"separator":61},[51,5046,1067],{},[89,5048,5049],{"encoding":91},"k, v",[29,5051,5053],{"className":5052,"ariaHidden":61},[96],[29,5054,5056,5059,5062,5065,5068],{"className":5055},[100],[29,5057],{"className":5058,"style":556},[104],[29,5060,1045],{"className":5061,"style":1387},[109,113],[29,5063,62],{"className":5064},[167],[29,5066],{"className":5067,"style":172},[171],[29,5069,1067],{"className":5070,"style":1319},[109,113]," 的上标 ",[29,5073,5075,5092],{"className":5074},[32],[29,5076,5078],{"className":5077},[36],[38,5079,5080],{"xmlns":40},[42,5081,5082,5090],{},[45,5083,5084,5086,5088],{},[59,5085,460],{"stretchy":459},[51,5087,463],{},[59,5089,466],{"stretchy":459},[89,5091,469],{"encoding":91},[29,5093,5095],{"className":5094,"ariaHidden":61},[96],[29,5096,5098,5101,5104,5107],{"className":5097},[100],[29,5099],{"className":5100,"style":479},[104],[29,5102,460],{"className":5103},[483],[29,5105,463],{"className":5106},[109,113],[29,5108,466],{"className":5109},[490]," 去掉。它是 2019 年提出来的。那会儿 LLM 还没火，可见减少 KV Cache 早就已经是热门课题了。KV Cache 直接降到原来的 ",[29,5112,5114,5132],{"className":5113},[32],[29,5115,5117],{"className":5116},[36],[38,5118,5119],{"xmlns":40},[42,5120,5121,5129],{},[45,5122,5123,5125,5127],{},[55,5124,57],{},[51,5126,3083],{"mathvariant":1002},[51,5128,428],{},[89,5130,5131],{"encoding":91},"1\u002Fh",[29,5133,5135],{"className":5134,"ariaHidden":61},[96],[29,5136,5138,5141,5145],{"className":5137},[100],[29,5139],{"className":5140,"style":479},[104],[29,5142,5144],{"className":5143},[109],"1\u002F",[29,5146,428],{"className":5147},[109,113],"。单从省显存的角度看，这已经是天花板了。",[10,5150,5151],{},"代价也很直白：原来一个 token 的每个 head 都有各自的 K、V，现在被压成一份，信息损失不小。",[10,5153,5154],{},"不过 MQA 的支持者认为这个损失被高估了。理由是不同 head 学到的 K、V 本来就高度相关，共享一份丢不掉太多有效信息；而且这部分损失可以通过继续训练补回来。这个论证后来被证明有一定道理，但\"一定道理\"和\"可以无条件依赖\"是两回事。GQA 的出现本身就说明，业界并不敢把压缩率开到 MQA 那么极端。",[10,5156,5157,5158,5186,5187,5216],{},"有个细节容易被忽略。MQA 共享 K、V 之后，attention 部分的参数量少了将近一半。为了保持总参数量不变，通常会相应增大 FFN 的规模。这能补回来一部分效果损失。但还有人担心压得太狠，影响学习效率和最终效果。于是有了 GQA（Grouped-Query Attention），那是 2023 年的事。GQA 把 ",[29,5159,5161,5174],{"className":5160},[32],[29,5162,5164],{"className":5163},[36],[38,5165,5166],{"xmlns":40},[42,5167,5168,5172],{},[45,5169,5170],{},[51,5171,428],{},[89,5173,428],{"encoding":91},[29,5175,5177],{"className":5176,"ariaHidden":61},[96],[29,5178,5180,5183],{"className":5179},[100],[29,5181],{"className":5182,"style":440},[104],[29,5184,428],{"className":5185},[109,113]," 个 head 分成 ",[29,5188,5190,5204],{"className":5189},[32],[29,5191,5193],{"className":5192},[36],[38,5194,5195],{"xmlns":40},[42,5196,5197,5202],{},[45,5198,5199],{},[51,5200,5201],{},"g",[89,5203,5201],{"encoding":91},[29,5205,5207],{"className":5206,"ariaHidden":61},[96],[29,5208,5210,5213],{"className":5209},[100],[29,5211],{"className":5212,"style":105},[104],[29,5214,5201],{"className":5215,"style":1319},[109,113]," 组，每组共享同一对 K、V。具体怎么分？",[10,5218,5219,5220,5248,5249,5295],{},"记第 ",[29,5221,5223,5236],{"className":5222},[32],[29,5224,5226],{"className":5225},[36],[38,5227,5228],{"xmlns":40},[42,5229,5230,5234],{},[45,5231,5232],{},[51,5233,463],{},[89,5235,463],{"encoding":91},[29,5237,5239],{"className":5238,"ariaHidden":61},[96],[29,5240,5242,5245],{"className":5241},[100],[29,5243],{"className":5244,"style":537},[104],[29,5246,463],{"className":5247},[109,113]," 个 head 属于第 ",[29,5250,5252,5273],{"className":5251},[32],[29,5253,5255],{"className":5254},[36],[38,5256,5257],{"xmlns":40},[42,5258,5259,5270],{},[45,5260,5261,5264,5266,5268],{},[51,5262,5263],{},"γ",[59,5265,460],{"stretchy":459},[51,5267,463],{},[59,5269,466],{"stretchy":459},[89,5271,5272],{"encoding":91},"\\gamma(s)",[29,5274,5276],{"className":5275,"ariaHidden":61},[96],[29,5277,5279,5282,5286,5289,5292],{"className":5278},[100],[29,5280],{"className":5281,"style":479},[104],[29,5283,5263],{"className":5284,"style":5285},[109,113],"margin-right:0.0556em;",[29,5287,460],{"className":5288},[483],[29,5290,463],{"className":5291},[109,113],[29,5293,466],{"className":5294},[490]," 组：",[29,5297,5299],{"className":5298},[597],[29,5300,5302,5342],{"className":5301},[32],[29,5303,5305],{"className":5304},[36],[38,5306,5307],{"xmlns":40,"display":606},[42,5308,5309,5339],{},[45,5310,5311,5313,5315,5317,5319,5321],{},[51,5312,5263],{},[59,5314,460],{"stretchy":459},[51,5316,463],{},[59,5318,466],{"stretchy":459},[59,5320,508],{},[45,5322,5323,5326,5336],{},[59,5324,5325],{"fence":61},"⌈",[1088,5327,5328,5334],{},[45,5329,5330,5332],{},[51,5331,463],{},[51,5333,5201],{},[51,5335,428],{},[59,5337,5338],{"fence":61},"⌉",[89,5340,5341],{"encoding":91},"\\gamma(s) = \\left\\lceil \\frac{sg}{h} \\right\\rceil",[29,5343,5345,5372],{"className":5344,"ariaHidden":61},[96],[29,5346,5348,5351,5354,5357,5360,5363,5366,5369],{"className":5347},[100],[29,5349],{"className":5350,"style":479},[104],[29,5352,5263],{"className":5353,"style":5285},[109,113],[29,5355,460],{"className":5356},[483],[29,5358,463],{"className":5359},[109,113],[29,5361,466],{"className":5362},[490],[29,5364],{"className":5365,"style":367},[171],[29,5367,508],{"className":5368},[371],[29,5370],{"className":5371,"style":367},[171],[29,5373,5375,5379],{"className":5374},[100],[29,5376],{"className":5377,"style":5378},[104],"height:1.836em;vertical-align:-0.686em;",[29,5380,5382,5392,5463],{"className":5381},[222],[29,5383,5387],{"className":5384,"style":5386},[483,5385],"delimcenter","top:0em;",[29,5388,5325],{"className":5389},[5390,5391],"delimsizing","size2",[29,5393,5395,5398,5460],{"className":5394},[109],[29,5396],{"className":5397},[483,1549],[29,5399,5401],{"className":5400},[1088],[29,5402,5404,5451],{"className":5403},[121,122],[29,5405,5407,5448],{"className":5406},[126],[29,5408,5411,5424,5433],{"className":5409,"style":5410},[130],"height:1.1076em;",[29,5412,5414,5418],{"style":5413},"top:-2.314em;",[29,5415],{"className":5416,"style":5417},[138],"height:3em;",[29,5419,5421],{"className":5420},[109],[29,5422,428],{"className":5423},[109,113],[29,5425,5427,5430],{"style":5426},"top:-3.23em;",[29,5428],{"className":5429,"style":5417},[138],[29,5431],{"className":5432,"style":1776},[1775],[29,5434,5436,5439],{"style":5435},"top:-3.677em;",[29,5437],{"className":5438,"style":5417},[138],[29,5440,5442,5445],{"className":5441},[109],[29,5443,463],{"className":5444},[109,113],[29,5446,5201],{"className":5447,"style":1319},[109,113],[29,5449,154],{"className":5450},[153],[29,5452,5454],{"className":5453},[126],[29,5455,5458],{"className":5456,"style":5457},[130],"height:0.686em;",[29,5459],{},[29,5461],{"className":5462},[490,1549],[29,5464,5466],{"className":5465,"style":5386},[490,5385],[29,5467,5338],{"className":5468},[5390,5391],[10,5470,5471,5472,5511,5512,5550],{},"（就是按顺序平均切开：前 ",[29,5473,5475,5493],{"className":5474},[32],[29,5476,5478],{"className":5477},[36],[38,5479,5480],{"xmlns":40},[42,5481,5482,5490],{},[45,5483,5484,5486,5488],{},[51,5485,428],{},[51,5487,3083],{"mathvariant":1002},[51,5489,5201],{},[89,5491,5492],{"encoding":91},"h\u002Fg",[29,5494,5496],{"className":5495,"ariaHidden":61},[96],[29,5497,5499,5502,5505,5508],{"className":5498},[100],[29,5500],{"className":5501,"style":479},[104],[29,5503,428],{"className":5504},[109,113],[29,5506,3083],{"className":5507},[109],[29,5509,5201],{"className":5510,"style":1319},[109,113]," 个 head 一组，接着 ",[29,5513,5515,5532],{"className":5514},[32],[29,5516,5518],{"className":5517},[36],[38,5519,5520],{"xmlns":40},[42,5521,5522,5530],{},[45,5523,5524,5526,5528],{},[51,5525,428],{},[51,5527,3083],{"mathvariant":1002},[51,5529,5201],{},[89,5531,5492],{"encoding":91},[29,5533,5535],{"className":5534,"ariaHidden":61},[96],[29,5536,5538,5541,5544,5547],{"className":5537},[100],[29,5539],{"className":5540,"style":479},[104],[29,5542,428],{"className":5543},[109,113],[29,5545,3083],{"className":5546},[109],[29,5548,5201],{"className":5549,"style":1319},[109,113]," 个一组，以此类推。）于是 attention 写成：",[29,5552,5554],{"className":5553},[597],[29,5555,5557,5682],{"className":5556},[32],[29,5558,5560],{"className":5559},[36],[38,5561,5562],{"xmlns":40,"display":606},[42,5563,5564,5679],{},[45,5565,5566,5580,5582,5602],{},[625,5567,5568,5570,5572],{},[51,5569,615],{},[51,5571,618],{},[45,5573,5574,5576,5578],{},[59,5575,460],{"stretchy":459},[51,5577,463],{},[59,5579,466],{"stretchy":459},[59,5581,508],{},[45,5583,5584,5586,5588,5590,5592,5594,5596,5598,5600],{},[51,5585,1003],{"mathvariant":1002},[51,5587,618],{"mathvariant":1002},[51,5589,618],{"mathvariant":1002},[51,5591,1010],{"mathvariant":1002},[51,5593,1013],{"mathvariant":1002},[51,5595,618],{"mathvariant":1002},[51,5597,296],{"mathvariant":1002},[51,5599,615],{"mathvariant":1002},[51,5601,1013],{"mathvariant":1002},[45,5603,5604,5606,5620,5622,5625,5649,5651,5653,5677],{},[59,5605,460],{"fence":61},[625,5607,5608,5610,5612],{},[51,5609,1028],{},[51,5611,618],{},[45,5613,5614,5616,5618],{},[59,5615,460],{"stretchy":459},[51,5617,463],{},[59,5619,466],{"stretchy":459},[59,5621,62],{"separator":61},[76,5623,5624],{}," ",[625,5626,5627,5629,5635],{},[51,5628,1045],{},[45,5630,5631,5633],{},[59,5632,1050],{},[51,5634,618],{},[45,5636,5637,5639,5641,5643,5645,5647],{},[59,5638,460],{"stretchy":459},[51,5640,5263],{},[59,5642,460],{"stretchy":459},[51,5644,463],{},[59,5646,466],{"stretchy":459},[59,5648,466],{"stretchy":459},[59,5650,62],{"separator":61},[76,5652,5624],{},[625,5654,5655,5657,5663],{},[51,5656,1067],{},[45,5658,5659,5661],{},[59,5660,1050],{},[51,5662,618],{},[45,5664,5665,5667,5669,5671,5673,5675],{},[59,5666,460],{"stretchy":459},[51,5668,5263],{},[59,5670,460],{"stretchy":459},[51,5672,463],{},[59,5674,466],{"stretchy":459},[59,5676,466],{"stretchy":459},[59,5678,466],{"fence":61},[89,5680,5681],{"encoding":91},"o_t^{(s)} = \\mathrm{Attention}\\left(q_t^{(s)},\\ k^{(\\gamma(s))}_{\\le t},\\ v^{(\\gamma(s))}_{\\le t}\\right)",[29,5683,5685,5760],{"className":5684,"ariaHidden":61},[96],[29,5686,5688,5691,5751,5754,5757],{"className":5687},[100],[29,5689],{"className":5690,"style":1225},[104],[29,5692,5694,5697],{"className":5693},[109],[29,5695,615],{"className":5696},[109,113],[29,5698,5700],{"className":5699},[117],[29,5701,5703,5743],{"className":5702},[121,122],[29,5704,5706,5740],{"className":5705},[126],[29,5707,5709,5720],{"className":5708,"style":771},[130],[29,5710,5711,5714],{"style":774},[29,5712],{"className":5713,"style":139},[138],[29,5715,5717],{"className":5716},[143,144,145,146],[29,5718,618],{"className":5719},[109,113,146],[29,5721,5722,5725],{"style":786},[29,5723],{"className":5724,"style":139},[138],[29,5726,5728],{"className":5727},[143,144,145,146],[29,5729,5731,5734,5737],{"className":5730},[109,146],[29,5732,460],{"className":5733},[483,146],[29,5735,463],{"className":5736},[109,113,146],[29,5738,466],{"className":5739},[490,146],[29,5741,154],{"className":5742},[153],[29,5744,5746],{"className":5745},[126],[29,5747,5749],{"className":5748,"style":814},[130],[29,5750],{},[29,5752],{"className":5753,"style":367},[171],[29,5755,508],{"className":5756},[371],[29,5758],{"className":5759,"style":367},[171],[29,5761,5763,5767,5773,5776],{"className":5762},[100],[29,5764],{"className":5765,"style":5766},[104],"height:1.8em;vertical-align:-0.65em;",[29,5768,5770],{"className":5769},[109],[29,5771,1309],{"className":5772},[109,1308],[29,5774],{"className":5775,"style":172},[171],[29,5777,5779,5785,5845,5848,5851,5854,5927,5930,5933,5936,6008],{"className":5778},[222],[29,5780,5782],{"className":5781,"style":5386},[483,5385],[29,5783,460],{"className":5784},[5390,5391],[29,5786,5788,5791],{"className":5787},[109],[29,5789,1028],{"className":5790,"style":1319},[109,113],[29,5792,5794],{"className":5793},[117],[29,5795,5797,5837],{"className":5796},[121,122],[29,5798,5800,5834],{"className":5799},[126],[29,5801,5803,5814],{"className":5802,"style":771},[130],[29,5804,5805,5808],{"style":1334},[29,5806],{"className":5807,"style":139},[138],[29,5809,5811],{"className":5810},[143,144,145,146],[29,5812,618],{"className":5813},[109,113,146],[29,5815,5816,5819],{"style":786},[29,5817],{"className":5818,"style":139},[138],[29,5820,5822],{"className":5821},[143,144,145,146],[29,5823,5825,5828,5831],{"className":5824},[109,146],[29,5826,460],{"className":5827},[483,146],[29,5829,463],{"className":5830},[109,113,146],[29,5832,466],{"className":5833},[490,146],[29,5835,154],{"className":5836},[153],[29,5838,5840],{"className":5839},[126],[29,5841,5843],{"className":5842,"style":814},[130],[29,5844],{},[29,5846,62],{"className":5847},[167],[29,5849,5624],{"className":5850},[171],[29,5852],{"className":5853,"style":172},[171],[29,5855,5857,5860],{"className":5856},[109],[29,5858,1045],{"className":5859,"style":1387},[109,113],[29,5861,5863],{"className":5862},[117],[29,5864,5866,5919],{"className":5865},[121,122],[29,5867,5869,5916],{"className":5868},[126],[29,5870,5872,5889],{"className":5871,"style":771},[130],[29,5873,5874,5877],{"style":1402},[29,5875],{"className":5876,"style":139},[138],[29,5878,5880],{"className":5879},[143,144,145,146],[29,5881,5883,5886],{"className":5882},[109,146],[29,5884,1050],{"className":5885},[371,146],[29,5887,618],{"className":5888},[109,113,146],[29,5890,5891,5894],{"style":786},[29,5892],{"className":5893,"style":139},[138],[29,5895,5897],{"className":5896},[143,144,145,146],[29,5898,5900,5903,5906,5909,5912],{"className":5899},[109,146],[29,5901,460],{"className":5902},[483,146],[29,5904,5263],{"className":5905,"style":5285},[109,113,146],[29,5907,460],{"className":5908},[483,146],[29,5910,463],{"className":5911},[109,113,146],[29,5913,5915],{"className":5914},[490,146],"))",[29,5917,154],{"className":5918},[153],[29,5920,5922],{"className":5921},[126],[29,5923,5925],{"className":5924,"style":1447},[130],[29,5926],{},[29,5928,62],{"className":5929},[167],[29,5931,5624],{"className":5932},[171],[29,5934],{"className":5935,"style":172},[171],[29,5937,5939,5942],{"className":5938},[109],[29,5940,1067],{"className":5941,"style":1319},[109,113],[29,5943,5945],{"className":5944},[117],[29,5946,5948,6000],{"className":5947},[121,122],[29,5949,5951,5997],{"className":5950},[126],[29,5952,5954,5971],{"className":5953,"style":771},[130],[29,5955,5956,5959],{"style":1476},[29,5957],{"className":5958,"style":139},[138],[29,5960,5962],{"className":5961},[143,144,145,146],[29,5963,5965,5968],{"className":5964},[109,146],[29,5966,1050],{"className":5967},[371,146],[29,5969,618],{"className":5970},[109,113,146],[29,5972,5973,5976],{"style":786},[29,5974],{"className":5975,"style":139},[138],[29,5977,5979],{"className":5978},[143,144,145,146],[29,5980,5982,5985,5988,5991,5994],{"className":5981},[109,146],[29,5983,460],{"className":5984},[483,146],[29,5986,5263],{"className":5987,"style":5285},[109,113,146],[29,5989,460],{"className":5990},[483,146],[29,5992,463],{"className":5993},[109,113,146],[29,5995,5915],{"className":5996},[490,146],[29,5998,154],{"className":5999},[153],[29,6001,6003],{"className":6002},[126],[29,6004,6006],{"className":6005,"style":1447},[130],[29,6007],{},[29,6009,6011],{"className":6010,"style":5386},[490,5385],[29,6012,466],{"className":6013},[5390,5391],[10,6015,6016,6017,6069,6070,6121,6122,6173,6174,6225],{},"注意 Q 仍然是每个 head 一份，只有 K、V 被分组共用。这个不对称就是 GQA 的全部内容。举个具体的数。",[29,6018,6020,6039],{"className":6019},[32],[29,6021,6023],{"className":6022},[36],[38,6024,6025],{"xmlns":40},[42,6026,6027,6036],{},[45,6028,6029,6031,6033],{},[51,6030,428],{},[59,6032,508],{},[55,6034,6035],{},"8",[89,6037,6038],{"encoding":91},"h = 8",[29,6040,6042,6060],{"className":6041,"ariaHidden":61},[96],[29,6043,6045,6048,6051,6054,6057],{"className":6044},[100],[29,6046],{"className":6047,"style":440},[104],[29,6049,428],{"className":6050},[109,113],[29,6052],{"className":6053,"style":367},[171],[29,6055,508],{"className":6056},[371],[29,6058],{"className":6059,"style":367},[171],[29,6061,6063,6066],{"className":6062},[100],[29,6064],{"className":6065,"style":3272},[104],[29,6067,6035],{"className":6068},[109],"、",[29,6071,6073,6091],{"className":6072},[32],[29,6074,6076],{"className":6075},[36],[38,6077,6078],{"xmlns":40},[42,6079,6080,6088],{},[45,6081,6082,6084,6086],{},[51,6083,5201],{},[59,6085,508],{},[55,6087,69],{},[89,6089,6090],{"encoding":91},"g = 2",[29,6092,6094,6112],{"className":6093,"ariaHidden":61},[96],[29,6095,6097,6100,6103,6106,6109],{"className":6096},[100],[29,6098],{"className":6099,"style":105},[104],[29,6101,5201],{"className":6102,"style":1319},[109,113],[29,6104],{"className":6105,"style":367},[171],[29,6107,508],{"className":6108},[371],[29,6110],{"className":6111,"style":367},[171],[29,6113,6115,6118],{"className":6114},[100],[29,6116],{"className":6117,"style":3272},[104],[29,6119,69],{"className":6120},[109]," 时，第 1 到第 4 个 head 共用一对 K、V，第 5 到第 8 个 head 共用另一对。",[29,6123,6125,6143],{"className":6124},[32],[29,6126,6128],{"className":6127},[36],[38,6129,6130],{"xmlns":40},[42,6131,6132,6140],{},[45,6133,6134,6136,6138],{},[51,6135,5201],{},[59,6137,508],{},[51,6139,428],{},[89,6141,6142],{"encoding":91},"g = h",[29,6144,6146,6164],{"className":6145,"ariaHidden":61},[96],[29,6147,6149,6152,6155,6158,6161],{"className":6148},[100],[29,6150],{"className":6151,"style":105},[104],[29,6153,5201],{"className":6154,"style":1319},[109,113],[29,6156],{"className":6157,"style":367},[171],[29,6159,508],{"className":6160},[371],[29,6162],{"className":6163,"style":367},[171],[29,6165,6167,6170],{"className":6166},[100],[29,6168],{"className":6169,"style":440},[104],[29,6171,428],{"className":6172},[109,113]," 时每个 head 各自一对，退化成 MHA；",[29,6175,6177,6195],{"className":6176},[32],[29,6178,6180],{"className":6179},[36],[38,6181,6182],{"xmlns":40},[42,6183,6184,6192],{},[45,6185,6186,6188,6190],{},[51,6187,5201],{},[59,6189,508],{},[55,6191,57],{},[89,6193,6194],{"encoding":91},"g = 1",[29,6196,6198,6216],{"className":6197,"ariaHidden":61},[96],[29,6199,6201,6204,6207,6210,6213],{"className":6200},[100],[29,6202],{"className":6203,"style":105},[104],[29,6205,5201],{"className":6206,"style":1319},[109,113],[29,6208],{"className":6209,"style":367},[171],[29,6211,508],{"className":6212},[371],[29,6214],{"className":6215,"style":367},[171],[29,6217,6219,6222],{"className":6218},[100],[29,6220],{"className":6221,"style":3272},[104],[29,6223,57],{"className":6224},[109]," 时所有 head 共用一对，退化成 MQA。",[10,6227,6228],{},"它提供了 MHA 到 MQA 的自然过渡，压缩率不如 MQA，但自由度和效果保证更好。",[10,6230,6231,6232,6283,6284,6334],{},"llama2\u002F3-70B 用的 ",[29,6233,6235,6253],{"className":6234},[32],[29,6236,6238],{"className":6237},[36],[38,6239,6240],{"xmlns":40},[42,6241,6242,6250],{},[45,6243,6244,6246,6248],{},[51,6245,5201],{},[59,6247,508],{},[55,6249,6035],{},[89,6251,6252],{"encoding":91},"g = 8",[29,6254,6256,6274],{"className":6255,"ariaHidden":61},[96],[29,6257,6259,6262,6265,6268,6271],{"className":6258},[100],[29,6260],{"className":6261,"style":105},[104],[29,6263,5201],{"className":6264,"style":1319},[109,113],[29,6266],{"className":6267,"style":367},[171],[29,6269,508],{"className":6270},[371],[29,6272],{"className":6273,"style":367},[171],[29,6275,6277,6280],{"className":6276},[100],[29,6278],{"className":6279,"style":3272},[104],[29,6281,6035],{"className":6282},[109],"。早年有个很整齐的解释：70B 这个体量放不进单卡，得单机 8 卡，而 attention 的 head 本来就是独立运算再拼接的，",[29,6285,6287,6304],{"className":6286},[32],[29,6288,6290],{"className":6289},[36],[38,6291,6292],{"xmlns":40},[42,6293,6294,6302],{},[45,6295,6296,6298,6300],{},[51,6297,5201],{},[59,6299,508],{},[55,6301,6035],{},[89,6303,6252],{"encoding":91},[29,6305,6307,6325],{"className":6306,"ariaHidden":61},[96],[29,6308,6310,6313,6316,6319,6322],{"className":6309},[100],[29,6311],{"className":6312,"style":105},[104],[29,6314,5201],{"className":6315,"style":1319},[109,113],[29,6317],{"className":6318,"style":367},[171],[29,6320,508],{"className":6321},[371],[29,6323],{"className":6324,"style":367},[171],[29,6326,6328,6331],{"className":6327},[100],[29,6329],{"className":6330,"style":3272},[104],[29,6332,6035],{"className":6333},[109]," 正好让每张卡负责一组 K、V 对应的 head，把卡间通信压到最小。这个解释现在不太站得住了。量化让更大的模型能进单卡，MoE 让 activated 参数远小于总参数，跨机推理也早就不是禁区。",[10,6336,6337,6338,6345],{},"两个现成的例子：Qwen3.8-Flash-Next 是 125B 的模型、每个 token 只激活 6B，在 B200 上用 TP4 就能跑（",[6339,6340,6344],"a",{"href":6341,"rel":6342},"https:\u002F\u002Fwww.lmsys.org\u002Fblog\u002F2026-08-26-qwen-flash-next",[6343],"nofollow","SGLang 的 day-0 支持","）；DeepSeek-V4-Flash-FP8 同样 TP4 就能跑，H200 上就够。",[10,6347,6348,6349,6377,6378,6428],{},"所以更准确的说法是：",[29,6350,6352,6365],{"className":6351},[32],[29,6353,6355],{"className":6354},[36],[38,6356,6357],{"xmlns":40},[42,6358,6359,6363],{},[45,6360,6361],{},[51,6362,5201],{},[89,6364,5201],{"encoding":91},[29,6366,6368],{"className":6367,"ariaHidden":61},[96],[29,6369,6371,6374],{"className":6370},[100],[29,6372],{"className":6373,"style":105},[104],[29,6375,5201],{"className":6376,"style":1319},[109,113]," 跟并行度对齐仍然是个好主意，但\"必须单机 8 卡\"这种硬约束已经没有了。",[29,6379,6381,6398],{"className":6380},[32],[29,6382,6384],{"className":6383},[36],[38,6385,6386],{"xmlns":40},[42,6387,6388,6396],{},[45,6389,6390,6392,6394],{},[51,6391,5201],{},[59,6393,508],{},[55,6395,6035],{},[89,6397,6252],{"encoding":91},[29,6399,6401,6419],{"className":6400,"ariaHidden":61},[96],[29,6402,6404,6407,6410,6413,6416],{"className":6403},[100],[29,6405],{"className":6406,"style":105},[104],[29,6408,5201],{"className":6409,"style":1319},[109,113],[29,6411],{"className":6412,"style":367},[171],[29,6414,508],{"className":6415},[371],[29,6417],{"className":6418,"style":367},[171],[29,6420,6422,6425],{"className":6421},[100],[29,6423],{"className":6424,"style":3272},[104],[29,6426,6035],{"className":6427},[109]," 能留下来，一半是因为并行度，一半是习惯。但 MQA 和 GQA 有一个共同的性质，值得单独点出来：",[10,6430,6431],{},[14,6432,6433],{},"它们压的是 head 的数量，不是 head 的内容。",[10,6435,6436,6437,6440],{},"被保留下来的那些 KV，每一个仍然是完整的、未经压缩的向量。共享是",[14,6438,6439],{},"离散","的。要么整份共用，要么各自一份，中间没有别的可能。下一节就是打破这个限制。",[21,6442,6444],{"id":6443},"mla","MLA",[10,6446,6447],{},"MLA（Multi-head Latent Attention）是 DeepSeek-V2 在 2024 年 5 月提出来的，到现在两年多。技术报告是从低秩投影的角度引入的，以至于有人问：LoRA 提出这么久了，为什么直到 MLA 才有人对 KV Cache 做低秩分解？我认为低秩投影这个角度不贴近本质。因为要说低秩投影的话，只要把 GQA 的所有 K、V 拼成一个长向量，就会发现 GQA 本身也是低秩投影。",[10,6449,6450,6451,6522,6523,6551],{},"把这个拼起来的向量记作 ",[29,6452,6454,6473],{"className":6453},[32],[29,6455,6457],{"className":6456},[36],[38,6458,6459],{"xmlns":40},[42,6460,6461,6470],{},[45,6462,6463],{},[48,6464,6465,6468],{},[51,6466,6467],{},"c",[51,6469,296],{},[89,6471,6472],{"encoding":91},"c_i",[29,6474,6476],{"className":6475,"ariaHidden":61},[96],[29,6477,6479,6482],{"className":6478},[100],[29,6480],{"className":6481,"style":692},[104],[29,6483,6485,6488],{"className":6484},[109],[29,6486,6467],{"className":6487},[109,113],[29,6489,6491],{"className":6490},[117],[29,6492,6494,6514],{"className":6493},[121,122],[29,6495,6497,6511],{"className":6496},[126],[29,6498,6500],{"className":6499,"style":341},[130],[29,6501,6502,6505],{"style":134},[29,6503],{"className":6504,"style":139},[138],[29,6506,6508],{"className":6507},[143,144,145,146],[29,6509,296],{"className":6510},[109,113,146],[29,6512,154],{"className":6513},[153],[29,6515,6517],{"className":6516},[126],[29,6518,6520],{"className":6519,"style":161},[130],[29,6521],{},"。它由 ",[29,6524,6526,6539],{"className":6525},[32],[29,6527,6529],{"className":6528},[36],[38,6530,6531],{"xmlns":40},[42,6532,6533,6537],{},[45,6534,6535],{},[51,6536,5201],{},[89,6538,5201],{"encoding":91},[29,6540,6542],{"className":6541,"ariaHidden":61},[96],[29,6543,6545,6548],{"className":6544},[100],[29,6546],{"className":6547,"style":105},[104],[29,6549,5201],{"className":6550,"style":1319},[109,113]," 组 K、V 依次首尾相接而成：",[29,6553,6555],{"className":6554},[597],[29,6556,6558,6684],{"className":6557},[32],[29,6559,6561],{"className":6560},[36],[38,6562,6563],{"xmlns":40,"display":606},[42,6564,6565,6681],{},[45,6566,6567,6573,6575,6577,6591,6593,6595,6597,6599,6613,6615,6629,6631,6633,6635,6637,6651,6653,6655],{},[48,6568,6569,6571],{},[51,6570,6467],{},[51,6572,296],{},[59,6574,508],{},[59,6576,623],{"stretchy":459},[625,6578,6579,6581,6583],{},[51,6580,1045],{},[51,6582,296],{},[45,6584,6585,6587,6589],{},[59,6586,460],{"stretchy":459},[55,6588,57],{},[59,6590,466],{"stretchy":459},[59,6592,62],{"separator":61},[59,6594,74],{},[76,6596,78],{},[59,6598,62],{"separator":61},[625,6600,6601,6603,6605],{},[51,6602,1045],{},[51,6604,296],{},[45,6606,6607,6609,6611],{},[59,6608,460],{"stretchy":459},[51,6610,5201],{},[59,6612,466],{"stretchy":459},[59,6614,62],{"separator":61},[625,6616,6617,6619,6621],{},[51,6618,1067],{},[51,6620,296],{},[45,6622,6623,6625,6627],{},[59,6624,460],{"stretchy":459},[55,6626,57],{},[59,6628,466],{"stretchy":459},[59,6630,62],{"separator":61},[59,6632,74],{},[76,6634,78],{},[59,6636,62],{"separator":61},[625,6638,6639,6641,6643],{},[51,6640,1067],{},[51,6642,296],{},[45,6644,6645,6647,6649],{},[59,6646,460],{"stretchy":459},[51,6648,5201],{},[59,6650,466],{"stretchy":459},[59,6652,679],{"stretchy":459},[59,6654,299],{},[301,6656,6657,6659],{},[51,6658,306],{"mathvariant":305},[45,6660,6661,6663,6665,6671,6673,6679],{},[51,6662,5201],{},[59,6664,460],{"stretchy":459},[48,6666,6667,6669],{},[51,6668,309],{},[51,6670,1045],{},[59,6672,3236],{},[48,6674,6675,6677],{},[51,6676,309],{},[51,6678,1067],{},[59,6680,466],{"stretchy":459},[89,6682,6683],{"encoding":91},"c_i = [k_i^{(1)}, \\cdots, k_i^{(g)}, v_i^{(1)}, \\cdots, v_i^{(g)}] \\in \\mathbb{R}^{g(d_k + d_v)}",[29,6685,6687,6742,7051],{"className":6686,"ariaHidden":61},[96],[29,6688,6690,6693,6733,6736,6739],{"className":6689},[100],[29,6691],{"className":6692,"style":692},[104],[29,6694,6696,6699],{"className":6695},[109],[29,6697,6467],{"className":6698},[109,113],[29,6700,6702],{"className":6701},[117],[29,6703,6705,6725],{"className":6704},[121,122],[29,6706,6708,6722],{"className":6707},[126],[29,6709,6711],{"className":6710,"style":341},[130],[29,6712,6713,6716],{"style":134},[29,6714],{"className":6715,"style":139},[138],[29,6717,6719],{"className":6718},[143,144,145,146],[29,6720,296],{"className":6721},[109,113,146],[29,6723,154],{"className":6724},[153],[29,6726,6728],{"className":6727},[126],[29,6729,6731],{"className":6730,"style":161},[130],[29,6732],{},[29,6734],{"className":6735,"style":367},[171],[29,6737,508],{"className":6738},[371],[29,6740],{"className":6741,"style":367},[171],[29,6743,6745,6748,6751,6811,6814,6817,6820,6823,6826,6829,6832,6892,6895,6898,6958,6961,6964,6967,6970,6973,6976,6979,7039,7042,7045,7048],{"className":6744},[100],[29,6746],{"className":6747,"style":2227},[104],[29,6749,623],{"className":6750},[483],[29,6752,6754,6757],{"className":6753},[109],[29,6755,1045],{"className":6756,"style":1387},[109,113],[29,6758,6760],{"className":6759},[117],[29,6761,6763,6803],{"className":6762},[121,122],[29,6764,6766,6800],{"className":6765},[126],[29,6767,6769,6780],{"className":6768,"style":771},[130],[29,6770,6771,6774],{"style":1718},[29,6772],{"className":6773,"style":139},[138],[29,6775,6777],{"className":6776},[143,144,145,146],[29,6778,296],{"className":6779},[109,113,146],[29,6781,6782,6785],{"style":786},[29,6783],{"className":6784,"style":139},[138],[29,6786,6788],{"className":6787},[143,144,145,146],[29,6789,6791,6794,6797],{"className":6790},[109,146],[29,6792,460],{"className":6793},[483,146],[29,6795,57],{"className":6796},[109,146],[29,6798,466],{"className":6799},[490,146],[29,6801,154],{"className":6802},[153],[29,6804,6806],{"className":6805},[126],[29,6807,6809],{"className":6808,"style":1760},[130],[29,6810],{},[29,6812,62],{"className":6813},[167],[29,6815],{"className":6816,"style":172},[171],[29,6818,74],{"className":6819},[222],[29,6821],{"className":6822,"style":172},[171],[29,6824],{"className":6825,"style":172},[171],[29,6827,62],{"className":6828},[167],[29,6830],{"className":6831,"style":172},[171],[29,6833,6835,6838],{"className":6834},[109],[29,6836,1045],{"className":6837,"style":1387},[109,113],[29,6839,6841],{"className":6840},[117],[29,6842,6844,6884],{"className":6843},[121,122],[29,6845,6847,6881],{"className":6846},[126],[29,6848,6850,6861],{"className":6849,"style":771},[130],[29,6851,6852,6855],{"style":1718},[29,6853],{"className":6854,"style":139},[138],[29,6856,6858],{"className":6857},[143,144,145,146],[29,6859,296],{"className":6860},[109,113,146],[29,6862,6863,6866],{"style":786},[29,6864],{"className":6865,"style":139},[138],[29,6867,6869],{"className":6868},[143,144,145,146],[29,6870,6872,6875,6878],{"className":6871},[109,146],[29,6873,460],{"className":6874},[483,146],[29,6876,5201],{"className":6877,"style":1319},[109,113,146],[29,6879,466],{"className":6880},[490,146],[29,6882,154],{"className":6883},[153],[29,6885,6887],{"className":6886},[126],[29,6888,6890],{"className":6889,"style":1760},[130],[29,6891],{},[29,6893,62],{"className":6894},[167],[29,6896],{"className":6897,"style":172},[171],[29,6899,6901,6904],{"className":6900},[109],[29,6902,1067],{"className":6903,"style":1319},[109,113],[29,6905,6907],{"className":6906},[117],[29,6908,6910,6950],{"className":6909},[121,122],[29,6911,6913,6947],{"className":6912},[126],[29,6914,6916,6927],{"className":6915,"style":771},[130],[29,6917,6918,6921],{"style":1990},[29,6919],{"className":6920,"style":139},[138],[29,6922,6924],{"className":6923},[143,144,145,146],[29,6925,296],{"className":6926},[109,113,146],[29,6928,6929,6932],{"style":786},[29,6930],{"className":6931,"style":139},[138],[29,6933,6935],{"className":6934},[143,144,145,146],[29,6936,6938,6941,6944],{"className":6937},[109,146],[29,6939,460],{"className":6940},[483,146],[29,6942,57],{"className":6943},[109,146],[29,6945,466],{"className":6946},[490,146],[29,6948,154],{"className":6949},[153],[29,6951,6953],{"className":6952},[126],[29,6954,6956],{"className":6955,"style":1760},[130],[29,6957],{},[29,6959,62],{"className":6960},[167],[29,6962],{"className":6963,"style":172},[171],[29,6965,74],{"className":6966},[222],[29,6968],{"className":6969,"style":172},[171],[29,6971],{"className":6972,"style":172},[171],[29,6974,62],{"className":6975},[167],[29,6977],{"className":6978,"style":172},[171],[29,6980,6982,6985],{"className":6981},[109],[29,6983,1067],{"className":6984,"style":1319},[109,113],[29,6986,6988],{"className":6987},[117],[29,6989,6991,7031],{"className":6990},[121,122],[29,6992,6994,7028],{"className":6993},[126],[29,6995,6997,7008],{"className":6996,"style":771},[130],[29,6998,6999,7002],{"style":1990},[29,7000],{"className":7001,"style":139},[138],[29,7003,7005],{"className":7004},[143,144,145,146],[29,7006,296],{"className":7007},[109,113,146],[29,7009,7010,7013],{"style":786},[29,7011],{"className":7012,"style":139},[138],[29,7014,7016],{"className":7015},[143,144,145,146],[29,7017,7019,7022,7025],{"className":7018},[109,146],[29,7020,460],{"className":7021},[483,146],[29,7023,5201],{"className":7024,"style":1319},[109,113,146],[29,7026,466],{"className":7027},[490,146],[29,7029,154],{"className":7030},[153],[29,7032,7034],{"className":7033},[126],[29,7035,7037],{"className":7036,"style":1760},[130],[29,7038],{},[29,7040,679],{"className":7041},[490],[29,7043],{"className":7044,"style":367},[171],[29,7046,299],{"className":7047},[371],[29,7049],{"className":7050,"style":367},[171],[29,7052,7054,7057],{"className":7053},[100],[29,7055],{"className":7056,"style":2363},[104],[29,7058,7060,7063],{"className":7059},[109],[29,7061,306],{"className":7062},[109,388],[29,7064,7066],{"className":7065},[117],[29,7067,7069],{"className":7068},[121],[29,7070,7072],{"className":7071},[126],[29,7073,7075],{"className":7074,"style":2363},[130],[29,7076,7077,7080],{"style":2378},[29,7078],{"className":7079,"style":139},[138],[29,7081,7083],{"className":7082},[143,144,145,146],[29,7084,7086,7089,7092,7132,7135,7175],{"className":7085},[109,146],[29,7087,5201],{"className":7088,"style":1319},[109,113,146],[29,7090,460],{"className":7091},[483,146],[29,7093,7095,7098],{"className":7094},[109,146],[29,7096,309],{"className":7097},[109,113,146],[29,7099,7101],{"className":7100},[117],[29,7102,7104,7124],{"className":7103},[121,122],[29,7105,7107,7121],{"className":7106},[126],[29,7108,7110],{"className":7109,"style":2472},[130],[29,7111,7112,7115],{"style":2475},[29,7113],{"className":7114,"style":2479},[138],[29,7116,7118],{"className":7117},[143,2483,2484,146],[29,7119,1045],{"className":7120,"style":1387},[109,113,146],[29,7122,154],{"className":7123},[153],[29,7125,7127],{"className":7126},[126],[29,7128,7130],{"className":7129,"style":2497},[130],[29,7131],{},[29,7133,3236],{"className":7134},[3262,146],[29,7136,7138,7141],{"className":7137},[109,146],[29,7139,309],{"className":7140},[109,113,146],[29,7142,7144],{"className":7143},[117],[29,7145,7147,7167],{"className":7146},[121,122],[29,7148,7150,7164],{"className":7149},[126],[29,7151,7153],{"className":7152,"style":3020},[130],[29,7154,7155,7158],{"style":3023},[29,7156],{"className":7157,"style":2479},[138],[29,7159,7161],{"className":7160},[143,2483,2484,146],[29,7162,1067],{"className":7163,"style":1319},[109,113,146],[29,7165,154],{"className":7166},[153],[29,7168,7170],{"className":7169},[126],[29,7171,7173],{"className":7172,"style":3042},[130],[29,7174],{},[29,7176,466],{"className":7177},[490,146],[10,7179,7180,7181,7251,7252,7323],{},"而它整个是由 ",[29,7182,7184,7202],{"className":7183},[32],[29,7185,7187],{"className":7186},[36],[38,7188,7189],{"xmlns":40},[42,7190,7191,7199],{},[45,7192,7193],{},[48,7194,7195,7197],{},[51,7196,53],{},[51,7198,296],{},[89,7200,7201],{"encoding":91},"x_i",[29,7203,7205],{"className":7204,"ariaHidden":61},[96],[29,7206,7208,7211],{"className":7207},[100],[29,7209],{"className":7210,"style":692},[104],[29,7212,7214,7217],{"className":7213},[109],[29,7215,53],{"className":7216},[109,113],[29,7218,7220],{"className":7219},[117],[29,7221,7223,7243],{"className":7222},[121,122],[29,7224,7226,7240],{"className":7225},[126],[29,7227,7229],{"className":7228,"style":341},[130],[29,7230,7231,7234],{"style":134},[29,7232],{"className":7233,"style":139},[138],[29,7235,7237],{"className":7236},[143,144,145,146],[29,7238,296],{"className":7239},[109,113,146],[29,7241,154],{"className":7242},[153],[29,7244,7246],{"className":7245},[126],[29,7247,7249],{"className":7248,"style":161},[130],[29,7250],{}," 一次矩阵乘法得到的。把各组投影矩阵横向拼起来记作 ",[29,7253,7255,7273],{"className":7254},[32],[29,7256,7258],{"className":7257},[36],[38,7259,7260],{"xmlns":40},[42,7261,7262,7270],{},[45,7263,7264],{},[48,7265,7266,7268],{},[51,7267,2087],{},[51,7269,6467],{},[89,7271,7272],{"encoding":91},"W_c",[29,7274,7276],{"className":7275,"ariaHidden":61},[96],[29,7277,7279,7282],{"className":7278},[100],[29,7280],{"className":7281,"style":4046},[104],[29,7283,7285,7288],{"className":7284},[109],[29,7286,2087],{"className":7287,"style":2350},[109,113],[29,7289,7291],{"className":7290},[117],[29,7292,7294,7315],{"className":7293},[121,122],[29,7295,7297,7312],{"className":7296},[126],[29,7298,7300],{"className":7299,"style":3172},[130],[29,7301,7303,7306],{"style":7302},"top:-2.55em;margin-left:-0.1389em;margin-right:0.05em;",[29,7304],{"className":7305,"style":139},[138],[29,7307,7309],{"className":7308},[143,144,145,146],[29,7310,6467],{"className":7311},[109,113,146],[29,7313,154],{"className":7314},[153],[29,7316,7318],{"className":7317},[126],[29,7319,7321],{"className":7320,"style":161},[130],[29,7322],{},"：",[29,7325,7327],{"className":7326},[597],[29,7328,7330,7485],{"className":7329},[32],[29,7331,7333],{"className":7332},[36],[38,7334,7335],{"xmlns":40,"display":606},[42,7336,7337,7482],{},[45,7338,7339,7345,7347,7353,7359,7361,7364,7370,7372,7374,7388,7390,7392,7394,7396,7410,7412,7426,7428,7430,7432,7434,7448,7450,7452],{},[48,7340,7341,7343],{},[51,7342,6467],{},[51,7344,296],{},[59,7346,508],{},[48,7348,7349,7351],{},[51,7350,53],{},[51,7352,296],{},[48,7354,7355,7357],{},[51,7356,2087],{},[51,7358,6467],{},[59,7360,62],{"separator":61},[171,7362],{"width":7363},"2em",[48,7365,7366,7368],{},[51,7367,2087],{},[51,7369,6467],{},[59,7371,508],{},[59,7373,623],{"stretchy":459},[625,7375,7376,7378,7380],{},[51,7377,2087],{},[51,7379,1045],{},[45,7381,7382,7384,7386],{},[59,7383,460],{"stretchy":459},[55,7385,57],{},[59,7387,466],{"stretchy":459},[59,7389,62],{"separator":61},[59,7391,74],{},[76,7393,78],{},[59,7395,62],{"separator":61},[625,7397,7398,7400,7402],{},[51,7399,2087],{},[51,7401,1045],{},[45,7403,7404,7406,7408],{},[59,7405,460],{"stretchy":459},[51,7407,5201],{},[59,7409,466],{"stretchy":459},[59,7411,62],{"separator":61},[625,7413,7414,7416,7418],{},[51,7415,2087],{},[51,7417,1067],{},[45,7419,7420,7422,7424],{},[59,7421,460],{"stretchy":459},[55,7423,57],{},[59,7425,466],{"stretchy":459},[59,7427,62],{"separator":61},[59,7429,74],{},[76,7431,78],{},[59,7433,62],{"separator":61},[625,7435,7436,7438,7440],{},[51,7437,2087],{},[51,7439,1067],{},[45,7441,7442,7444,7446],{},[59,7443,460],{"stretchy":459},[51,7445,5201],{},[59,7447,466],{"stretchy":459},[59,7449,679],{"stretchy":459},[59,7451,299],{},[301,7453,7454,7456],{},[51,7455,306],{"mathvariant":305},[45,7457,7458,7460,7462,7464,7466,7472,7474,7480],{},[51,7459,309],{},[59,7461,3514],{},[51,7463,5201],{},[59,7465,460],{"stretchy":459},[48,7467,7468,7470],{},[51,7469,309],{},[51,7471,1045],{},[59,7473,3236],{},[48,7475,7476,7478],{},[51,7477,309],{},[51,7479,1067],{},[59,7481,466],{"stretchy":459},[89,7483,7484],{"encoding":91},"c_i = x_i W_c, \\qquad W_c = [W_k^{(1)}, \\cdots, W_k^{(g)}, W_v^{(1)}, \\cdots, W_v^{(g)}] \\in \\mathbb{R}^{d \\times g(d_k + d_v)}",[29,7486,7488,7543,7689,7998],{"className":7487,"ariaHidden":61},[96],[29,7489,7491,7494,7534,7537,7540],{"className":7490},[100],[29,7492],{"className":7493,"style":692},[104],[29,7495,7497,7500],{"className":7496},[109],[29,7498,6467],{"className":7499},[109,113],[29,7501,7503],{"className":7502},[117],[29,7504,7506,7526],{"className":7505},[121,122],[29,7507,7509,7523],{"className":7508},[126],[29,7510,7512],{"className":7511,"style":341},[130],[29,7513,7514,7517],{"style":134},[29,7515],{"className":7516,"style":139},[138],[29,7518,7520],{"className":7519},[143,144,145,146],[29,7521,296],{"className":7522},[109,113,146],[29,7524,154],{"className":7525},[153],[29,7527,7529],{"className":7528},[126],[29,7530,7532],{"className":7531,"style":161},[130],[29,7533],{},[29,7535],{"className":7536,"style":367},[171],[29,7538,508],{"className":7539},[371],[29,7541],{"className":7542,"style":367},[171],[29,7544,7546,7550,7590,7630,7633,7637,7640,7680,7683,7686],{"className":7545},[100],[29,7547],{"className":7548,"style":7549},[104],"height:0.8778em;vertical-align:-0.1944em;",[29,7551,7553,7556],{"className":7552},[109],[29,7554,53],{"className":7555},[109,113],[29,7557,7559],{"className":7558},[117],[29,7560,7562,7582],{"className":7561},[121,122],[29,7563,7565,7579],{"className":7564},[126],[29,7566,7568],{"className":7567,"style":341},[130],[29,7569,7570,7573],{"style":134},[29,7571],{"className":7572,"style":139},[138],[29,7574,7576],{"className":7575},[143,144,145,146],[29,7577,296],{"className":7578},[109,113,146],[29,7580,154],{"className":7581},[153],[29,7583,7585],{"className":7584},[126],[29,7586,7588],{"className":7587,"style":161},[130],[29,7589],{},[29,7591,7593,7596],{"className":7592},[109],[29,7594,2087],{"className":7595,"style":2350},[109,113],[29,7597,7599],{"className":7598},[117],[29,7600,7602,7622],{"className":7601},[121,122],[29,7603,7605,7619],{"className":7604},[126],[29,7606,7608],{"className":7607,"style":3172},[130],[29,7609,7610,7613],{"style":7302},[29,7611],{"className":7612,"style":139},[138],[29,7614,7616],{"className":7615},[143,144,145,146],[29,7617,6467],{"className":7618},[109,113,146],[29,7620,154],{"className":7621},[153],[29,7623,7625],{"className":7624},[126],[29,7626,7628],{"className":7627,"style":161},[130],[29,7629],{},[29,7631,62],{"className":7632},[167],[29,7634],{"className":7635,"style":7636},[171],"margin-right:2em;",[29,7638],{"className":7639,"style":172},[171],[29,7641,7643,7646],{"className":7642},[109],[29,7644,2087],{"className":7645,"style":2350},[109,113],[29,7647,7649],{"className":7648},[117],[29,7650,7652,7672],{"className":7651},[121,122],[29,7653,7655,7669],{"className":7654},[126],[29,7656,7658],{"className":7657,"style":3172},[130],[29,7659,7660,7663],{"style":7302},[29,7661],{"className":7662,"style":139},[138],[29,7664,7666],{"className":7665},[143,144,145,146],[29,7667,6467],{"className":7668},[109,113,146],[29,7670,154],{"className":7671},[153],[29,7673,7675],{"className":7674},[126],[29,7676,7678],{"className":7677,"style":161},[130],[29,7679],{},[29,7681],{"className":7682,"style":367},[171],[29,7684,508],{"className":7685},[371],[29,7687],{"className":7688,"style":367},[171],[29,7690,7692,7695,7698,7758,7761,7764,7767,7770,7773,7776,7779,7839,7842,7845,7905,7908,7911,7914,7917,7920,7923,7926,7986,7989,7992,7995],{"className":7691},[100],[29,7693],{"className":7694,"style":2585},[104],[29,7696,623],{"className":7697},[483],[29,7699,7701,7704],{"className":7700},[109],[29,7702,2087],{"className":7703,"style":2350},[109,113],[29,7705,7707],{"className":7706},[117],[29,7708,7710,7750],{"className":7709},[121,122],[29,7711,7713,7747],{"className":7712},[126],[29,7714,7716,7727],{"className":7715,"style":771},[130],[29,7717,7718,7721],{"style":2646},[29,7719],{"className":7720,"style":139},[138],[29,7722,7724],{"className":7723},[143,144,145,146],[29,7725,1045],{"className":7726,"style":1387},[109,113,146],[29,7728,7729,7732],{"style":786},[29,7730],{"className":7731,"style":139},[138],[29,7733,7735],{"className":7734},[143,144,145,146],[29,7736,7738,7741,7744],{"className":7737},[109,146],[29,7739,460],{"className":7740},[483,146],[29,7742,57],{"className":7743},[109,146],[29,7745,466],{"className":7746},[490,146],[29,7748,154],{"className":7749},[153],[29,7751,7753],{"className":7752},[126],[29,7754,7756],{"className":7755,"style":2685},[130],[29,7757],{},[29,7759,62],{"className":7760},[167],[29,7762],{"className":7763,"style":172},[171],[29,7765,74],{"className":7766},[222],[29,7768],{"className":7769,"style":172},[171],[29,7771],{"className":7772,"style":172},[171],[29,7774,62],{"className":7775},[167],[29,7777],{"className":7778,"style":172},[171],[29,7780,7782,7785],{"className":7781},[109],[29,7783,2087],{"className":7784,"style":2350},[109,113],[29,7786,7788],{"className":7787},[117],[29,7789,7791,7831],{"className":7790},[121,122],[29,7792,7794,7828],{"className":7793},[126],[29,7795,7797,7808],{"className":7796,"style":771},[130],[29,7798,7799,7802],{"style":2646},[29,7800],{"className":7801,"style":139},[138],[29,7803,7805],{"className":7804},[143,144,145,146],[29,7806,1045],{"className":7807,"style":1387},[109,113,146],[29,7809,7810,7813],{"style":786},[29,7811],{"className":7812,"style":139},[138],[29,7814,7816],{"className":7815},[143,144,145,146],[29,7817,7819,7822,7825],{"className":7818},[109,146],[29,7820,460],{"className":7821},[483,146],[29,7823,5201],{"className":7824,"style":1319},[109,113,146],[29,7826,466],{"className":7827},[490,146],[29,7829,154],{"className":7830},[153],[29,7832,7834],{"className":7833},[126],[29,7835,7837],{"className":7836,"style":2685},[130],[29,7838],{},[29,7840,62],{"className":7841},[167],[29,7843],{"className":7844,"style":172},[171],[29,7846,7848,7851],{"className":7847},[109],[29,7849,2087],{"className":7850,"style":2350},[109,113],[29,7852,7854],{"className":7853},[117],[29,7855,7857,7897],{"className":7856},[121,122],[29,7858,7860,7894],{"className":7859},[126],[29,7861,7863,7874],{"className":7862,"style":2363},[130],[29,7864,7865,7868],{"style":2366},[29,7866],{"className":7867,"style":139},[138],[29,7869,7871],{"className":7870},[143,144,145,146],[29,7872,1067],{"className":7873,"style":1319},[109,113,146],[29,7875,7876,7879],{"style":2378},[29,7877],{"className":7878,"style":139},[138],[29,7880,7882],{"className":7881},[143,144,145,146],[29,7883,7885,7888,7891],{"className":7884},[109,146],[29,7886,460],{"className":7887},[483,146],[29,7889,57],{"className":7890},[109,146],[29,7892,466],{"className":7893},[490,146],[29,7895,154],{"className":7896},[153],[29,7898,7900],{"className":7899},[126],[29,7901,7903],{"className":7902,"style":2955},[130],[29,7904],{},[29,7906,62],{"className":7907},[167],[29,7909],{"className":7910,"style":172},[171],[29,7912,74],{"className":7913},[222],[29,7915],{"className":7916,"style":172},[171],[29,7918],{"className":7919,"style":172},[171],[29,7921,62],{"className":7922},[167],[29,7924],{"className":7925,"style":172},[171],[29,7927,7929,7932],{"className":7928},[109],[29,7930,2087],{"className":7931,"style":2350},[109,113],[29,7933,7935],{"className":7934},[117],[29,7936,7938,7978],{"className":7937},[121,122],[29,7939,7941,7975],{"className":7940},[126],[29,7942,7944,7955],{"className":7943,"style":2363},[130],[29,7945,7946,7949],{"style":2366},[29,7947],{"className":7948,"style":139},[138],[29,7950,7952],{"className":7951},[143,144,145,146],[29,7953,1067],{"className":7954,"style":1319},[109,113,146],[29,7956,7957,7960],{"style":2378},[29,7958],{"className":7959,"style":139},[138],[29,7961,7963],{"className":7962},[143,144,145,146],[29,7964,7966,7969,7972],{"className":7965},[109,146],[29,7967,460],{"className":7968},[483,146],[29,7970,5201],{"className":7971,"style":1319},[109,113,146],[29,7973,466],{"className":7974},[490,146],[29,7976,154],{"className":7977},[153],[29,7979,7981],{"className":7980},[126],[29,7982,7984],{"className":7983,"style":2955},[130],[29,7985],{},[29,7987,679],{"className":7988},[490],[29,7990],{"className":7991,"style":367},[171],[29,7993,299],{"className":7994},[371],[29,7996],{"className":7997,"style":367},[171],[29,7999,8001,8004],{"className":8000},[100],[29,8002],{"className":8003,"style":2363},[104],[29,8005,8007,8010],{"className":8006},[109],[29,8008,306],{"className":8009},[109,388],[29,8011,8013],{"className":8012},[117],[29,8014,8016],{"className":8015},[121],[29,8017,8019],{"className":8018},[126],[29,8020,8022],{"className":8021,"style":2363},[130],[29,8023,8024,8027],{"style":2378},[29,8025],{"className":8026,"style":139},[138],[29,8028,8030],{"className":8029},[143,144,145,146],[29,8031,8033,8036,8039,8042,8045,8085,8088,8128],{"className":8032},[109,146],[29,8034,309],{"className":8035},[109,113,146],[29,8037,3514],{"className":8038},[3262,146],[29,8040,5201],{"className":8041,"style":1319},[109,113,146],[29,8043,460],{"className":8044},[483,146],[29,8046,8048,8051],{"className":8047},[109,146],[29,8049,309],{"className":8050},[109,113,146],[29,8052,8054],{"className":8053},[117],[29,8055,8057,8077],{"className":8056},[121,122],[29,8058,8060,8074],{"className":8059},[126],[29,8061,8063],{"className":8062,"style":2472},[130],[29,8064,8065,8068],{"style":2475},[29,8066],{"className":8067,"style":2479},[138],[29,8069,8071],{"className":8070},[143,2483,2484,146],[29,8072,1045],{"className":8073,"style":1387},[109,113,146],[29,8075,154],{"className":8076},[153],[29,8078,8080],{"className":8079},[126],[29,8081,8083],{"className":8082,"style":2497},[130],[29,8084],{},[29,8086,3236],{"className":8087},[3262,146],[29,8089,8091,8094],{"className":8090},[109,146],[29,8092,309],{"className":8093},[109,113,146],[29,8095,8097],{"className":8096},[117],[29,8098,8100,8120],{"className":8099},[121,122],[29,8101,8103,8117],{"className":8102},[126],[29,8104,8106],{"className":8105,"style":3020},[130],[29,8107,8108,8111],{"style":3023},[29,8109],{"className":8110,"style":2479},[138],[29,8112,8114],{"className":8113},[143,2483,2484,146],[29,8115,1067],{"className":8116,"style":1319},[109,113,146],[29,8118,154],{"className":8119},[153],[29,8121,8123],{"className":8122},[126],[29,8124,8126],{"className":8125,"style":3042},[130],[29,8127],{},[29,8129,466],{"className":8130},[490,146],[10,8132,8133,8134,8368,8369,8503,8504,8507,8508,8536,8537,8575,8576,8604],{},"因为 ",[29,8135,8137,8182],{"className":8136},[32],[29,8138,8140],{"className":8139},[36],[38,8141,8142],{"xmlns":40},[42,8143,8144,8179],{},[45,8145,8146,8152,8154,8156,8158,8164,8166,8172,8174,8177],{},[48,8147,8148,8150],{},[51,8149,309],{},[51,8151,6467],{},[59,8153,508],{},[51,8155,5201],{},[59,8157,460],{"stretchy":459},[48,8159,8160,8162],{},[51,8161,309],{},[51,8163,1045],{},[59,8165,3236],{},[48,8167,8168,8170],{},[51,8169,309],{},[51,8171,1067],{},[59,8173,466],{"stretchy":459},[59,8175,8176],{},"\u003C",[51,8178,309],{},[89,8180,8181],{"encoding":91},"d_c = g(d_k + d_v) \u003C d",[29,8183,8185,8240,8301,8359],{"className":8184,"ariaHidden":61},[96],[29,8186,8188,8191,8231,8234,8237],{"className":8187},[100],[29,8189],{"className":8190,"style":3098},[104],[29,8192,8194,8197],{"className":8193},[109],[29,8195,309],{"className":8196},[109,113],[29,8198,8200],{"className":8199},[117],[29,8201,8203,8223],{"className":8202},[121,122],[29,8204,8206,8220],{"className":8205},[126],[29,8207,8209],{"className":8208,"style":3172},[130],[29,8210,8211,8214],{"style":134},[29,8212],{"className":8213,"style":139},[138],[29,8215,8217],{"className":8216},[143,144,145,146],[29,8218,6467],{"className":8219},[109,113,146],[29,8221,154],{"className":8222},[153],[29,8224,8226],{"className":8225},[126],[29,8227,8229],{"className":8228,"style":161},[130],[29,8230],{},[29,8232],{"className":8233,"style":367},[171],[29,8235,508],{"className":8236},[371],[29,8238],{"className":8239,"style":367},[171],[29,8241,8243,8246,8249,8252,8292,8295,8298],{"className":8242},[100],[29,8244],{"className":8245,"style":479},[104],[29,8247,5201],{"className":8248,"style":1319},[109,113],[29,8250,460],{"className":8251},[483],[29,8253,8255,8258],{"className":8254},[109],[29,8256,309],{"className":8257},[109,113],[29,8259,8261],{"className":8260},[117],[29,8262,8264,8284],{"className":8263},[121,122],[29,8265,8267,8281],{"className":8266},[126],[29,8268,8270],{"className":8269,"style":253},[130],[29,8271,8272,8275],{"style":134},[29,8273],{"className":8274,"style":139},[138],[29,8276,8278],{"className":8277},[143,144,145,146],[29,8279,1045],{"className":8280,"style":1387},[109,113,146],[29,8282,154],{"className":8283},[153],[29,8285,8287],{"className":8286},[126],[29,8288,8290],{"className":8289,"style":161},[130],[29,8291],{},[29,8293],{"className":8294,"style":3258},[171],[29,8296,3236],{"className":8297},[3262],[29,8299],{"className":8300,"style":3258},[171],[29,8302,8304,8307,8347,8350,8353,8356],{"className":8303},[100],[29,8305],{"className":8306,"style":479},[104],[29,8308,8310,8313],{"className":8309},[109],[29,8311,309],{"className":8312},[109,113],[29,8314,8316],{"className":8315},[117],[29,8317,8319,8339],{"className":8318},[121,122],[29,8320,8322,8336],{"className":8321},[126],[29,8323,8325],{"className":8324,"style":3172},[130],[29,8326,8327,8330],{"style":134},[29,8328],{"className":8329,"style":139},[138],[29,8331,8333],{"className":8332},[143,144,145,146],[29,8334,1067],{"className":8335,"style":1319},[109,113,146],[29,8337,154],{"className":8338},[153],[29,8340,8342],{"className":8341},[126],[29,8343,8345],{"className":8344,"style":161},[130],[29,8346],{},[29,8348,466],{"className":8349},[490],[29,8351],{"className":8352,"style":367},[171],[29,8354,8176],{"className":8355},[371],[29,8357],{"className":8358,"style":367},[171],[29,8360,8362,8365],{"className":8361},[100],[29,8363],{"className":8364,"style":440},[104],[29,8366,309],{"className":8367},[109,113],"，这一步就是在把维度压小。所以 ",[29,8370,8372,8399],{"className":8371},[32],[29,8373,8375],{"className":8374},[36],[38,8376,8377],{"xmlns":40},[42,8378,8379,8396],{},[45,8380,8381,8387,8390],{},[48,8382,8383,8385],{},[51,8384,53],{},[51,8386,296],{},[59,8388,8389],{},"→",[48,8391,8392,8394],{},[51,8393,6467],{},[51,8395,296],{},[89,8397,8398],{"encoding":91},"x_i \\to c_i",[29,8400,8402,8457],{"className":8401,"ariaHidden":61},[96],[29,8403,8405,8408,8448,8451,8454],{"className":8404},[100],[29,8406],{"className":8407,"style":692},[104],[29,8409,8411,8414],{"className":8410},[109],[29,8412,53],{"className":8413},[109,113],[29,8415,8417],{"className":8416},[117],[29,8418,8420,8440],{"className":8419},[121,122],[29,8421,8423,8437],{"className":8422},[126],[29,8424,8426],{"className":8425,"style":341},[130],[29,8427,8428,8431],{"style":134},[29,8429],{"className":8430,"style":139},[138],[29,8432,8434],{"className":8433},[143,144,145,146],[29,8435,296],{"className":8436},[109,113,146],[29,8438,154],{"className":8439},[153],[29,8441,8443],{"className":8442},[126],[29,8444,8446],{"className":8445,"style":161},[130],[29,8447],{},[29,8449],{"className":8450,"style":367},[171],[29,8452,8389],{"className":8453},[371],[29,8455],{"className":8456,"style":367},[171],[29,8458,8460,8463],{"className":8459},[100],[29,8461],{"className":8462,"style":692},[104],[29,8464,8466,8469],{"className":8465},[109],[29,8467,6467],{"className":8468},[109,113],[29,8470,8472],{"className":8471},[117],[29,8473,8475,8495],{"className":8474},[121,122],[29,8476,8478,8492],{"className":8477},[126],[29,8479,8481],{"className":8480,"style":341},[130],[29,8482,8483,8486],{"style":134},[29,8484],{"className":8485,"style":139},[138],[29,8487,8489],{"className":8488},[143,144,145,146],[29,8490,296],{"className":8491},[109,113,146],[29,8493,154],{"className":8494},[153],[29,8496,8498],{"className":8497},[126],[29,8499,8501],{"className":8500,"style":161},[130],[29,8502],{}," 本来就是低秩投影。也就是说，光看\"低秩投影\"这一步，GQA 和 MLA 没有区别。MLA 的改进落在",[14,8505,8506],{},"低秩投影之后做了什么","。GQA 投影之后干的事很简单：把向量对半分成 K 和 V，每一份再均分成 ",[29,8509,8511,8524],{"className":8510},[32],[29,8512,8514],{"className":8513},[36],[38,8515,8516],{"xmlns":40},[42,8517,8518,8522],{},[45,8519,8520],{},[51,8521,5201],{},[89,8523,5201],{"encoding":91},[29,8525,8527],{"className":8526,"ariaHidden":61},[96],[29,8528,8530,8533],{"className":8529},[100],[29,8531],{"className":8532,"style":105},[104],[29,8534,5201],{"className":8535,"style":1319},[109,113]," 份，每份复制 ",[29,8538,8540,8557],{"className":8539},[32],[29,8541,8543],{"className":8542},[36],[38,8544,8545],{"xmlns":40},[42,8546,8547,8555],{},[45,8548,8549,8551,8553],{},[51,8550,428],{},[51,8552,3083],{"mathvariant":1002},[51,8554,5201],{},[89,8556,5492],{"encoding":91},[29,8558,8560],{"className":8559,"ariaHidden":61},[96],[29,8561,8563,8566,8569,8572],{"className":8562},[100],[29,8564],{"className":8565,"style":479},[104],[29,8567,428],{"className":8568},[109,113],[29,8570,3083],{"className":8571},[109],[29,8573,5201],{"className":8574,"style":1319},[109,113]," 次，凑够 ",[29,8577,8579,8592],{"className":8578},[32],[29,8580,8582],{"className":8581},[36],[38,8583,8584],{"xmlns":40},[42,8585,8586,8590],{},[45,8587,8588],{},[51,8589,428],{},[89,8591,428],{"encoding":91},[29,8593,8595],{"className":8594,"ariaHidden":61},[96],[29,8596,8598,8601],{"className":8597},[100],[29,8599],{"className":8600,"style":440},[104],[29,8602,428],{"className":8603},[109,113]," 个 head。分割和复制都是极简的线性变换。",[10,8606,8607],{},"MLA 的第一步想法就是把它们换成一般的线性变换，让每个 head 拿到的 K、V 各不相同：",[29,8609,8611],{"className":8610},[597],[29,8612,8614,8726],{"className":8613},[32],[29,8615,8617],{"className":8616},[36],[38,8618,8619],{"xmlns":40,"display":606},[42,8620,8621,8723],{},[45,8622,8623,8637,8639,8645,8659,8661,8671,8673,8675,8689,8691,8697,8711,8713],{},[625,8624,8625,8627,8629],{},[51,8626,1045],{},[51,8628,296],{},[45,8630,8631,8633,8635],{},[59,8632,460],{"stretchy":459},[51,8634,463],{},[59,8636,466],{"stretchy":459},[59,8638,508],{},[48,8640,8641,8643],{},[51,8642,6467],{},[51,8644,296],{},[625,8646,8647,8649,8651],{},[51,8648,2087],{},[51,8650,1045],{},[45,8652,8653,8655,8657],{},[59,8654,460],{"stretchy":459},[51,8656,463],{},[59,8658,466],{"stretchy":459},[59,8660,299],{},[301,8662,8663,8665],{},[51,8664,306],{"mathvariant":305},[48,8666,8667,8669],{},[51,8668,309],{},[51,8670,1045],{},[59,8672,62],{"separator":61},[171,8674],{"width":2114},[625,8676,8677,8679,8681],{},[51,8678,1067],{},[51,8680,296],{},[45,8682,8683,8685,8687],{},[59,8684,460],{"stretchy":459},[51,8686,463],{},[59,8688,466],{"stretchy":459},[59,8690,508],{},[48,8692,8693,8695],{},[51,8694,6467],{},[51,8696,296],{},[625,8698,8699,8701,8703],{},[51,8700,2087],{},[51,8702,1067],{},[45,8704,8705,8707,8709],{},[59,8706,460],{"stretchy":459},[51,8708,463],{},[59,8710,466],{"stretchy":459},[59,8712,299],{},[301,8714,8715,8717],{},[51,8716,306],{"mathvariant":305},[48,8718,8719,8721],{},[51,8720,309],{},[51,8722,1067],{},[89,8724,8725],{"encoding":91},"k_i^{(s)} = c_i W_k^{(s)} \\in \\mathbb{R}^{d_k}, \\quad v_i^{(s)} = c_i W_v^{(s)} \\in \\mathbb{R}^{d_v}",[29,8727,8729,8804,8919,9072,9187],{"className":8728,"ariaHidden":61},[96],[29,8730,8732,8735,8795,8798,8801],{"className":8731},[100],[29,8733],{"className":8734,"style":2227},[104],[29,8736,8738,8741],{"className":8737},[109],[29,8739,1045],{"className":8740,"style":1387},[109,113],[29,8742,8744],{"className":8743},[117],[29,8745,8747,8787],{"className":8746},[121,122],[29,8748,8750,8784],{"className":8749},[126],[29,8751,8753,8764],{"className":8752,"style":771},[130],[29,8754,8755,8758],{"style":1718},[29,8756],{"className":8757,"style":139},[138],[29,8759,8761],{"className":8760},[143,144,145,146],[29,8762,296],{"className":8763},[109,113,146],[29,8765,8766,8769],{"style":786},[29,8767],{"className":8768,"style":139},[138],[29,8770,8772],{"className":8771},[143,144,145,146],[29,8773,8775,8778,8781],{"className":8774},[109,146],[29,8776,460],{"className":8777},[483,146],[29,8779,463],{"className":8780},[109,113,146],[29,8782,466],{"className":8783},[490,146],[29,8785,154],{"className":8786},[153],[29,8788,8790],{"className":8789},[126],[29,8791,8793],{"className":8792,"style":1760},[130],[29,8794],{},[29,8796],{"className":8797,"style":367},[171],[29,8799,508],{"className":8800},[371],[29,8802],{"className":8803,"style":367},[171],[29,8805,8807,8810,8850,8910,8913,8916],{"className":8806},[100],[29,8808],{"className":8809,"style":2585},[104],[29,8811,8813,8816],{"className":8812},[109],[29,8814,6467],{"className":8815},[109,113],[29,8817,8819],{"className":8818},[117],[29,8820,8822,8842],{"className":8821},[121,122],[29,8823,8825,8839],{"className":8824},[126],[29,8826,8828],{"className":8827,"style":341},[130],[29,8829,8830,8833],{"style":134},[29,8831],{"className":8832,"style":139},[138],[29,8834,8836],{"className":8835},[143,144,145,146],[29,8837,296],{"className":8838},[109,113,146],[29,8840,154],{"className":8841},[153],[29,8843,8845],{"className":8844},[126],[29,8846,8848],{"className":8847,"style":161},[130],[29,8849],{},[29,8851,8853,8856],{"className":8852},[109],[29,8854,2087],{"className":8855,"style":2350},[109,113],[29,8857,8859],{"className":8858},[117],[29,8860,8862,8902],{"className":8861},[121,122],[29,8863,8865,8899],{"className":8864},[126],[29,8866,8868,8879],{"className":8867,"style":771},[130],[29,8869,8870,8873],{"style":2646},[29,8871],{"className":8872,"style":139},[138],[29,8874,8876],{"className":8875},[143,144,145,146],[29,8877,1045],{"className":8878,"style":1387},[109,113,146],[29,8880,8881,8884],{"style":786},[29,8882],{"className":8883,"style":139},[138],[29,8885,8887],{"className":8886},[143,144,145,146],[29,8888,8890,8893,8896],{"className":8889},[109,146],[29,8891,460],{"className":8892},[483,146],[29,8894,463],{"className":8895},[109,113,146],[29,8897,466],{"className":8898},[490,146],[29,8900,154],{"className":8901},[153],[29,8903,8905],{"className":8904},[126],[29,8906,8908],{"className":8907,"style":2685},[130],[29,8909],{},[29,8911],{"className":8912,"style":367},[171],[29,8914,299],{"className":8915},[371],[29,8917],{"className":8918,"style":367},[171],[29,8920,8922,8925,8994,8997,9000,9003,9063,9066,9069],{"className":8921},[100],[29,8923],{"className":8924,"style":2227},[104],[29,8926,8928,8931],{"className":8927},[109],[29,8929,306],{"className":8930},[109,388],[29,8932,8934],{"className":8933},[117],[29,8935,8937],{"className":8936},[121],[29,8938,8940],{"className":8939},[126],[29,8941,8943],{"className":8942,"style":2442},[130],[29,8944,8945,8948],{"style":2378},[29,8946],{"className":8947,"style":139},[138],[29,8949,8951],{"className":8950},[143,144,145,146],[29,8952,8954],{"className":8953},[109,146],[29,8955,8957,8960],{"className":8956},[109,146],[29,8958,309],{"className":8959},[109,113,146],[29,8961,8963],{"className":8962},[117],[29,8964,8966,8986],{"className":8965},[121,122],[29,8967,8969,8983],{"className":8968},[126],[29,8970,8972],{"className":8971,"style":2472},[130],[29,8973,8974,8977],{"style":2475},[29,8975],{"className":8976,"style":2479},[138],[29,8978,8980],{"className":8979},[143,2483,2484,146],[29,8981,1045],{"className":8982,"style":1387},[109,113,146],[29,8984,154],{"className":8985},[153],[29,8987,8989],{"className":8988},[126],[29,8990,8992],{"className":8991,"style":2497},[130],[29,8993],{},[29,8995,62],{"className":8996},[167],[29,8998],{"className":8999,"style":2506},[171],[29,9001],{"className":9002,"style":172},[171],[29,9004,9006,9009],{"className":9005},[109],[29,9007,1067],{"className":9008,"style":1319},[109,113],[29,9010,9012],{"className":9011},[117],[29,9013,9015,9055],{"className":9014},[121,122],[29,9016,9018,9052],{"className":9017},[126],[29,9019,9021,9032],{"className":9020,"style":771},[130],[29,9022,9023,9026],{"style":1990},[29,9024],{"className":9025,"style":139},[138],[29,9027,9029],{"className":9028},[143,144,145,146],[29,9030,296],{"className":9031},[109,113,146],[29,9033,9034,9037],{"style":786},[29,9035],{"className":9036,"style":139},[138],[29,9038,9040],{"className":9039},[143,144,145,146],[29,9041,9043,9046,9049],{"className":9042},[109,146],[29,9044,460],{"className":9045},[483,146],[29,9047,463],{"className":9048},[109,113,146],[29,9050,466],{"className":9051},[490,146],[29,9053,154],{"className":9054},[153],[29,9056,9058],{"className":9057},[126],[29,9059,9061],{"className":9060,"style":1760},[130],[29,9062],{},[29,9064],{"className":9065,"style":367},[171],[29,9067,508],{"className":9068},[371],[29,9070],{"className":9071,"style":367},[171],[29,9073,9075,9078,9118,9178,9181,9184],{"className":9074},[100],[29,9076],{"className":9077,"style":2856},[104],[29,9079,9081,9084],{"className":9080},[109],[29,9082,6467],{"className":9083},[109,113],[29,9085,9087],{"className":9086},[117],[29,9088,9090,9110],{"className":9089},[121,122],[29,9091,9093,9107],{"className":9092},[126],[29,9094,9096],{"className":9095,"style":341},[130],[29,9097,9098,9101],{"style":134},[29,9099],{"className":9100,"style":139},[138],[29,9102,9104],{"className":9103},[143,144,145,146],[29,9105,296],{"className":9106},[109,113,146],[29,9108,154],{"className":9109},[153],[29,9111,9113],{"className":9112},[126],[29,9114,9116],{"className":9115,"style":161},[130],[29,9117],{},[29,9119,9121,9124],{"className":9120},[109],[29,9122,2087],{"className":9123,"style":2350},[109,113],[29,9125,9127],{"className":9126},[117],[29,9128,9130,9170],{"className":9129},[121,122],[29,9131,9133,9167],{"className":9132},[126],[29,9134,9136,9147],{"className":9135,"style":2363},[130],[29,9137,9138,9141],{"style":2366},[29,9139],{"className":9140,"style":139},[138],[29,9142,9144],{"className":9143},[143,144,145,146],[29,9145,1067],{"className":9146,"style":1319},[109,113,146],[29,9148,9149,9152],{"style":2378},[29,9150],{"className":9151,"style":139},[138],[29,9153,9155],{"className":9154},[143,144,145,146],[29,9156,9158,9161,9164],{"className":9157},[109,146],[29,9159,460],{"className":9160},[483,146],[29,9162,463],{"className":9163},[109,113,146],[29,9165,466],{"className":9166},[490,146],[29,9168,154],{"className":9169},[153],[29,9171,9173],{"className":9172},[126],[29,9174,9176],{"className":9175,"style":2955},[130],[29,9177],{},[29,9179],{"className":9180,"style":367},[171],[29,9182,299],{"className":9183},[371],[29,9185],{"className":9186,"style":367},[171],[29,9188,9190,9193],{"className":9189},[100],[29,9191],{"className":9192,"style":2442},[104],[29,9194,9196,9199],{"className":9195},[109],[29,9197,306],{"className":9198},[109,388],[29,9200,9202],{"className":9201},[117],[29,9203,9205],{"className":9204},[121],[29,9206,9208],{"className":9207},[126],[29,9209,9211],{"className":9210,"style":2442},[130],[29,9212,9213,9216],{"style":2378},[29,9214],{"className":9215,"style":139},[138],[29,9217,9219],{"className":9218},[143,144,145,146],[29,9220,9222],{"className":9221},[109,146],[29,9223,9225,9228],{"className":9224},[109,146],[29,9226,309],{"className":9227},[109,113,146],[29,9229,9231],{"className":9230},[117],[29,9232,9234,9254],{"className":9233},[121,122],[29,9235,9237,9251],{"className":9236},[126],[29,9238,9240],{"className":9239,"style":3020},[130],[29,9241,9242,9245],{"style":3023},[29,9243],{"className":9244,"style":2479},[138],[29,9246,9248],{"className":9247},[143,2483,2484,146],[29,9249,1067],{"className":9250,"style":1319},[109,113,146],[29,9252,154],{"className":9253},[153],[29,9255,9257],{"className":9256},[126],[29,9258,9260],{"className":9259,"style":3042},[130],[29,9261],{},[10,9263,9264,9265,9391],{},"但这样问题就来了。我们缓存的是投影后的 ",[29,9266,9268,9294],{"className":9267},[32],[29,9269,9271],{"className":9270},[36],[38,9272,9273],{"xmlns":40},[42,9274,9275,9291],{},[45,9276,9277,9283,9285],{},[48,9278,9279,9281],{},[51,9280,1045],{},[51,9282,296],{},[59,9284,62],{"separator":61},[48,9286,9287,9289],{},[51,9288,1067],{},[51,9290,296],{},[89,9292,9293],{"encoding":91},"k_i, v_i",[29,9295,9297],{"className":9296,"ariaHidden":61},[96],[29,9298,9300,9303,9344,9347,9350],{"className":9299},[100],[29,9301],{"className":9302,"style":556},[104],[29,9304,9306,9309],{"className":9305},[109],[29,9307,1045],{"className":9308,"style":1387},[109,113],[29,9310,9312],{"className":9311},[117],[29,9313,9315,9336],{"className":9314},[121,122],[29,9316,9318,9333],{"className":9317},[126],[29,9319,9321],{"className":9320,"style":341},[130],[29,9322,9324,9327],{"style":9323},"top:-2.55em;margin-left:-0.0315em;margin-right:0.05em;",[29,9325],{"className":9326,"style":139},[138],[29,9328,9330],{"className":9329},[143,144,145,146],[29,9331,296],{"className":9332},[109,113,146],[29,9334,154],{"className":9335},[153],[29,9337,9339],{"className":9338},[126],[29,9340,9342],{"className":9341,"style":161},[130],[29,9343],{},[29,9345,62],{"className":9346},[167],[29,9348],{"className":9349,"style":172},[171],[29,9351,9353,9356],{"className":9352},[109],[29,9354,1067],{"className":9355,"style":1319},[109,113],[29,9357,9359],{"className":9358},[117],[29,9360,9362,9383],{"className":9361},[121,122],[29,9363,9365,9380],{"className":9364},[126],[29,9366,9368],{"className":9367,"style":341},[130],[29,9369,9371,9374],{"style":9370},"top:-2.55em;margin-left:-0.0359em;margin-right:0.05em;",[29,9372],{"className":9373,"style":139},[138],[29,9375,9377],{"className":9376},[143,144,145,146],[29,9378,296],{"className":9379},[109,113,146],[29,9381,154],{"className":9382},[153],[29,9384,9386],{"className":9385},[126],[29,9387,9389],{"className":9388,"style":161},[130],[29,9390],{},"，一旦它们各不相同，KV Cache 就恢复成 MHA 那么大，GQA 的初衷全丢了。MLA 的做法是找一个恒等变换绕过去。在推理阶段：",[29,9393,9395],{"className":9394},[597],[29,9396,9398,9544],{"className":9397},[32],[29,9399,9401],{"className":9400},[36],[38,9402,9403],{"xmlns":40,"display":606},[42,9404,9405,9541],{},[45,9406,9407,9421,9437,9439,9441,9447,9461,9463,9465,9471,9485,9491,9493,9499,9501,9515,9531,9533],{},[625,9408,9409,9411,9413],{},[51,9410,1028],{},[51,9412,618],{},[45,9414,9415,9417,9419],{},[59,9416,460],{"stretchy":459},[51,9418,463],{},[59,9420,466],{"stretchy":459},[625,9422,9423,9425,9427],{},[51,9424,1045],{},[51,9426,296],{},[45,9428,9429,9431,9433,9435],{},[59,9430,460],{"stretchy":459},[51,9432,463],{},[59,9434,466],{"stretchy":459},[51,9436,1144],{"mathvariant":1002},[59,9438,508],{},[59,9440,460],{"stretchy":459},[48,9442,9443,9445],{},[51,9444,53],{},[51,9446,618],{},[625,9448,9449,9451,9453],{},[51,9450,2087],{},[51,9452,1028],{},[45,9454,9455,9457,9459],{},[59,9456,460],{"stretchy":459},[51,9458,463],{},[59,9460,466],{"stretchy":459},[59,9462,466],{"stretchy":459},[59,9464,460],{"stretchy":459},[48,9466,9467,9469],{},[51,9468,6467],{},[51,9470,296],{},[625,9472,9473,9475,9477],{},[51,9474,2087],{},[51,9476,1045],{},[45,9478,9479,9481,9483],{},[59,9480,460],{"stretchy":459},[51,9482,463],{},[59,9484,466],{"stretchy":459},[301,9486,9487,9489],{},[59,9488,466],{"stretchy":459},[51,9490,1144],{"mathvariant":1002},[59,9492,508],{},[48,9494,9495,9497],{},[51,9496,53],{},[51,9498,618],{},[59,9500,460],{"stretchy":459},[625,9502,9503,9505,9507],{},[51,9504,2087],{},[51,9506,1028],{},[45,9508,9509,9511,9513],{},[59,9510,460],{"stretchy":459},[51,9512,463],{},[59,9514,466],{"stretchy":459},[625,9516,9517,9519,9521],{},[51,9518,2087],{},[51,9520,1045],{},[45,9522,9523,9525,9527,9529],{},[59,9524,460],{"stretchy":459},[51,9526,463],{},[59,9528,466],{"stretchy":459},[51,9530,1144],{"mathvariant":1002},[59,9532,466],{"stretchy":459},[625,9534,9535,9537,9539],{},[51,9536,6467],{},[51,9538,296],{},[51,9540,1144],{"mathvariant":1002},[89,9542,9543],{"encoding":91},"q_t^{(s)} k_i^{(s)\\top} = (x_t W_q^{(s)})(c_i W_k^{(s)})^\\top = x_t (W_q^{(s)} W_k^{(s)\\top}) c_i^\\top",[29,9545,9547,9685,9939],{"className":9546,"ariaHidden":61},[96],[29,9548,9550,9553,9613,9676,9679,9682],{"className":9549},[100],[29,9551],{"className":9552,"style":2227},[104],[29,9554,9556,9559],{"className":9555},[109],[29,9557,1028],{"className":9558,"style":1319},[109,113],[29,9560,9562],{"className":9561},[117],[29,9563,9565,9605],{"className":9564},[121,122],[29,9566,9568,9602],{"className":9567},[126],[29,9569,9571,9582],{"className":9570,"style":771},[130],[29,9572,9573,9576],{"style":1334},[29,9574],{"className":9575,"style":139},[138],[29,9577,9579],{"className":9578},[143,144,145,146],[29,9580,618],{"className":9581},[109,113,146],[29,9583,9584,9587],{"style":786},[29,9585],{"className":9586,"style":139},[138],[29,9588,9590],{"className":9589},[143,144,145,146],[29,9591,9593,9596,9599],{"className":9592},[109,146],[29,9594,460],{"className":9595},[483,146],[29,9597,463],{"className":9598},[109,113,146],[29,9600,466],{"className":9601},[490,146],[29,9603,154],{"className":9604},[153],[29,9606,9608],{"className":9607},[126],[29,9609,9611],{"className":9610,"style":814},[130],[29,9612],{},[29,9614,9616,9619],{"className":9615},[109],[29,9617,1045],{"className":9618,"style":1387},[109,113],[29,9620,9622],{"className":9621},[117],[29,9623,9625,9668],{"className":9624},[121,122],[29,9626,9628,9665],{"className":9627},[126],[29,9629,9631,9642],{"className":9630,"style":771},[130],[29,9632,9633,9636],{"style":1718},[29,9634],{"className":9635,"style":139},[138],[29,9637,9639],{"className":9638},[143,144,145,146],[29,9640,296],{"className":9641},[109,113,146],[29,9643,9644,9647],{"style":786},[29,9645],{"className":9646,"style":139},[138],[29,9648,9650],{"className":9649},[143,144,145,146],[29,9651,9653,9656,9659,9662],{"className":9652},[109,146],[29,9654,460],{"className":9655},[483,146],[29,9657,463],{"className":9658},[109,113,146],[29,9660,466],{"className":9661},[490,146],[29,9663,1144],{"className":9664},[109,146],[29,9666,154],{"className":9667},[153],[29,9669,9671],{"className":9670},[126],[29,9672,9674],{"className":9673,"style":1760},[130],[29,9675],{},[29,9677],{"className":9678,"style":367},[171],[29,9680,508],{"className":9681},[371],[29,9683],{"className":9684,"style":367},[171],[29,9686,9688,9692,9695,9735,9795,9798,9801,9841,9901,9930,9933,9936],{"className":9687},[100],[29,9689],{"className":9690,"style":9691},[104],"height:1.4279em;vertical-align:-0.3831em;",[29,9693,460],{"className":9694},[483],[29,9696,9698,9701],{"className":9697},[109],[29,9699,53],{"className":9700},[109,113],[29,9702,9704],{"className":9703},[117],[29,9705,9707,9727],{"className":9706},[121,122],[29,9708,9710,9724],{"className":9709},[126],[29,9711,9713],{"className":9712,"style":711},[130],[29,9714,9715,9718],{"style":134},[29,9716],{"className":9717,"style":139},[138],[29,9719,9721],{"className":9720},[143,144,145,146],[29,9722,618],{"className":9723},[109,113,146],[29,9725,154],{"className":9726},[153],[29,9728,9730],{"className":9729},[126],[29,9731,9733],{"className":9732,"style":161},[130],[29,9734],{},[29,9736,9738,9741],{"className":9737},[109],[29,9739,2087],{"className":9740,"style":2350},[109,113],[29,9742,9744],{"className":9743},[117],[29,9745,9747,9787],{"className":9746},[121,122],[29,9748,9750,9784],{"className":9749},[126],[29,9751,9753,9764],{"className":9752,"style":2363},[130],[29,9754,9755,9758],{"style":2366},[29,9756],{"className":9757,"style":139},[138],[29,9759,9761],{"className":9760},[143,144,145,146],[29,9762,1028],{"className":9763,"style":1319},[109,113,146],[29,9765,9766,9769],{"style":2378},[29,9767],{"className":9768,"style":139},[138],[29,9770,9772],{"className":9771},[143,144,145,146],[29,9773,9775,9778,9781],{"className":9774},[109,146],[29,9776,460],{"className":9777},[483,146],[29,9779,463],{"className":9780},[109,113,146],[29,9782,466],{"className":9783},[490,146],[29,9785,154],{"className":9786},[153],[29,9788,9790],{"className":9789},[126],[29,9791,9793],{"className":9792,"style":2406},[130],[29,9794],{},[29,9796,466],{"className":9797},[490],[29,9799,460],{"className":9800},[483],[29,9802,9804,9807],{"className":9803},[109],[29,9805,6467],{"className":9806},[109,113],[29,9808,9810],{"className":9809},[117],[29,9811,9813,9833],{"className":9812},[121,122],[29,9814,9816,9830],{"className":9815},[126],[29,9817,9819],{"className":9818,"style":341},[130],[29,9820,9821,9824],{"style":134},[29,9822],{"className":9823,"style":139},[138],[29,9825,9827],{"className":9826},[143,144,145,146],[29,9828,296],{"className":9829},[109,113,146],[29,9831,154],{"className":9832},[153],[29,9834,9836],{"className":9835},[126],[29,9837,9839],{"className":9838,"style":161},[130],[29,9840],{},[29,9842,9844,9847],{"className":9843},[109],[29,9845,2087],{"className":9846,"style":2350},[109,113],[29,9848,9850],{"className":9849},[117],[29,9851,9853,9893],{"className":9852},[121,122],[29,9854,9856,9890],{"className":9855},[126],[29,9857,9859,9870],{"className":9858,"style":771},[130],[29,9860,9861,9864],{"style":2646},[29,9862],{"className":9863,"style":139},[138],[29,9865,9867],{"className":9866},[143,144,145,146],[29,9868,1045],{"className":9869,"style":1387},[109,113,146],[29,9871,9872,9875],{"style":786},[29,9873],{"className":9874,"style":139},[138],[29,9876,9878],{"className":9877},[143,144,145,146],[29,9879,9881,9884,9887],{"className":9880},[109,146],[29,9882,460],{"className":9883},[483,146],[29,9885,463],{"className":9886},[109,113,146],[29,9888,466],{"className":9889},[490,146],[29,9891,154],{"className":9892},[153],[29,9894,9896],{"className":9895},[126],[29,9897,9899],{"className":9898,"style":2685},[130],[29,9900],{},[29,9902,9904,9907],{"className":9903},[490],[29,9905,466],{"className":9906},[490],[29,9908,9910],{"className":9909},[117],[29,9911,9913],{"className":9912},[121],[29,9914,9916],{"className":9915},[126],[29,9917,9919],{"className":9918,"style":2442},[130],[29,9920,9921,9924],{"style":2378},[29,9922],{"className":9923,"style":139},[138],[29,9925,9927],{"className":9926},[143,144,145,146],[29,9928,1144],{"className":9929},[109,146],[29,9931],{"className":9932,"style":367},[171],[29,9934,508],{"className":9935},[371],[29,9937],{"className":9938,"style":367},[171],[29,9940,9942,9945,9985,9988,10048,10111,10114],{"className":9941},[100],[29,9943],{"className":9944,"style":9691},[104],[29,9946,9948,9951],{"className":9947},[109],[29,9949,53],{"className":9950},[109,113],[29,9952,9954],{"className":9953},[117],[29,9955,9957,9977],{"className":9956},[121,122],[29,9958,9960,9974],{"className":9959},[126],[29,9961,9963],{"className":9962,"style":711},[130],[29,9964,9965,9968],{"style":134},[29,9966],{"className":9967,"style":139},[138],[29,9969,9971],{"className":9970},[143,144,145,146],[29,9972,618],{"className":9973},[109,113,146],[29,9975,154],{"className":9976},[153],[29,9978,9980],{"className":9979},[126],[29,9981,9983],{"className":9982,"style":161},[130],[29,9984],{},[29,9986,460],{"className":9987},[483],[29,9989,9991,9994],{"className":9990},[109],[29,9992,2087],{"className":9993,"style":2350},[109,113],[29,9995,9997],{"className":9996},[117],[29,9998,10000,10040],{"className":9999},[121,122],[29,10001,10003,10037],{"className":10002},[126],[29,10004,10006,10017],{"className":10005,"style":2363},[130],[29,10007,10008,10011],{"style":2366},[29,10009],{"className":10010,"style":139},[138],[29,10012,10014],{"className":10013},[143,144,145,146],[29,10015,1028],{"className":10016,"style":1319},[109,113,146],[29,10018,10019,10022],{"style":2378},[29,10020],{"className":10021,"style":139},[138],[29,10023,10025],{"className":10024},[143,144,145,146],[29,10026,10028,10031,10034],{"className":10027},[109,146],[29,10029,460],{"className":10030},[483,146],[29,10032,463],{"className":10033},[109,113,146],[29,10035,466],{"className":10036},[490,146],[29,10038,154],{"className":10039},[153],[29,10041,10043],{"className":10042},[126],[29,10044,10046],{"className":10045,"style":2406},[130],[29,10047],{},[29,10049,10051,10054],{"className":10050},[109],[29,10052,2087],{"className":10053,"style":2350},[109,113],[29,10055,10057],{"className":10056},[117],[29,10058,10060,10103],{"className":10059},[121,122],[29,10061,10063,10100],{"className":10062},[126],[29,10064,10066,10077],{"className":10065,"style":771},[130],[29,10067,10068,10071],{"style":2646},[29,10069],{"className":10070,"style":139},[138],[29,10072,10074],{"className":10073},[143,144,145,146],[29,10075,1045],{"className":10076,"style":1387},[109,113,146],[29,10078,10079,10082],{"style":786},[29,10080],{"className":10081,"style":139},[138],[29,10083,10085],{"className":10084},[143,144,145,146],[29,10086,10088,10091,10094,10097],{"className":10087},[109,146],[29,10089,460],{"className":10090},[483,146],[29,10092,463],{"className":10093},[109,113,146],[29,10095,466],{"className":10096},[490,146],[29,10098,1144],{"className":10099},[109,146],[29,10101,154],{"className":10102},[153],[29,10104,10106],{"className":10105},[126],[29,10107,10109],{"className":10108,"style":2685},[130],[29,10110],{},[29,10112,466],{"className":10113},[490],[29,10115,10117,10120],{"className":10116},[109],[29,10118,6467],{"className":10119},[109,113],[29,10121,10123],{"className":10122},[117],[29,10124,10126,10158],{"className":10125},[121,122],[29,10127,10129,10155],{"className":10128},[126],[29,10130,10132,10144],{"className":10131,"style":2442},[130],[29,10133,10135,10138],{"style":10134},"top:-2.453em;margin-left:0em;margin-right:0.05em;",[29,10136],{"className":10137,"style":139},[138],[29,10139,10141],{"className":10140},[143,144,145,146],[29,10142,296],{"className":10143},[109,113,146],[29,10145,10146,10149],{"style":2378},[29,10147],{"className":10148,"style":139},[138],[29,10150,10152],{"className":10151},[143,144,145,146],[29,10153,1144],{"className":10154},[109,146],[29,10156,154],{"className":10157},[153],[29,10159,10161],{"className":10160},[126],[29,10162,10164],{"className":10163,"style":2955},[130],[29,10165],{},[10,10167,10168,10169,10348,10349,10418,10419,10489,10490,10727,10728,10827,10828,10898,10899,10968,10969,11038,11039,11108,11109,11147],{},"这意味着可以把 ",[29,10170,10172,10214],{"className":10171},[32],[29,10173,10175],{"className":10174},[36],[38,10176,10177],{"xmlns":40},[42,10178,10179,10211],{},[45,10180,10181,10195],{},[625,10182,10183,10185,10187],{},[51,10184,2087],{},[51,10186,1028],{},[45,10188,10189,10191,10193],{},[59,10190,460],{"stretchy":459},[51,10192,463],{},[59,10194,466],{"stretchy":459},[625,10196,10197,10199,10201],{},[51,10198,2087],{},[51,10200,1045],{},[45,10202,10203,10205,10207,10209],{},[59,10204,460],{"stretchy":459},[51,10206,463],{},[59,10208,466],{"stretchy":459},[51,10210,1144],{"mathvariant":1002},[89,10212,10213],{"encoding":91},"W_q^{(s)} W_k^{(s)\\top}",[29,10215,10217],{"className":10216,"ariaHidden":61},[96],[29,10218,10220,10223,10285],{"className":10219},[100],[29,10221],{"className":10222,"style":2585},[104],[29,10224,10226,10229],{"className":10225},[109],[29,10227,2087],{"className":10228,"style":2350},[109,113],[29,10230,10232],{"className":10231},[117],[29,10233,10235,10276],{"className":10234},[121,122],[29,10236,10238,10273],{"className":10237},[126],[29,10239,10241,10253],{"className":10240,"style":771},[130],[29,10242,10244,10247],{"style":10243},"top:-2.5834em;margin-left:-0.1389em;margin-right:0.05em;",[29,10245],{"className":10246,"style":139},[138],[29,10248,10250],{"className":10249},[143,144,145,146],[29,10251,1028],{"className":10252,"style":1319},[109,113,146],[29,10254,10255,10258],{"style":786},[29,10256],{"className":10257,"style":139},[138],[29,10259,10261],{"className":10260},[143,144,145,146],[29,10262,10264,10267,10270],{"className":10263},[109,146],[29,10265,460],{"className":10266},[483,146],[29,10268,463],{"className":10269},[109,113,146],[29,10271,466],{"className":10272},[490,146],[29,10274,154],{"className":10275},[153],[29,10277,10279],{"className":10278},[126],[29,10280,10283],{"className":10281,"style":10282},[130],"height:0.2527em;",[29,10284],{},[29,10286,10288,10291],{"className":10287},[109],[29,10289,2087],{"className":10290,"style":2350},[109,113],[29,10292,10294],{"className":10293},[117],[29,10295,10297,10340],{"className":10296},[121,122],[29,10298,10300,10337],{"className":10299},[126],[29,10301,10303,10314],{"className":10302,"style":771},[130],[29,10304,10305,10308],{"style":2646},[29,10306],{"className":10307,"style":139},[138],[29,10309,10311],{"className":10310},[143,144,145,146],[29,10312,1045],{"className":10313,"style":1387},[109,113,146],[29,10315,10316,10319],{"style":786},[29,10317],{"className":10318,"style":139},[138],[29,10320,10322],{"className":10321},[143,144,145,146],[29,10323,10325,10328,10331,10334],{"className":10324},[109,146],[29,10326,460],{"className":10327},[483,146],[29,10329,463],{"className":10330},[109,113,146],[29,10332,466],{"className":10333},[490,146],[29,10335,1144],{"className":10336},[109,146],[29,10338,154],{"className":10339},[153],[29,10341,10343],{"className":10342},[126],[29,10344,10346],{"className":10345,"style":2685},[130],[29,10347],{}," 合并起来当作 Q 的投影矩阵，那么 ",[29,10350,10352,10369],{"className":10351},[32],[29,10353,10355],{"className":10354},[36],[38,10356,10357],{"xmlns":40},[42,10358,10359,10367],{},[45,10360,10361],{},[48,10362,10363,10365],{},[51,10364,6467],{},[51,10366,296],{},[89,10368,6472],{"encoding":91},[29,10370,10372],{"className":10371,"ariaHidden":61},[96],[29,10373,10375,10378],{"className":10374},[100],[29,10376],{"className":10377,"style":692},[104],[29,10379,10381,10384],{"className":10380},[109],[29,10382,6467],{"className":10383},[109,113],[29,10385,10387],{"className":10386},[117],[29,10388,10390,10410],{"className":10389},[121,122],[29,10391,10393,10407],{"className":10392},[126],[29,10394,10396],{"className":10395,"style":341},[130],[29,10397,10398,10401],{"style":134},[29,10399],{"className":10400,"style":139},[138],[29,10402,10404],{"className":10403},[143,144,145,146],[29,10405,296],{"className":10406},[109,113,146],[29,10408,154],{"className":10409},[153],[29,10411,10413],{"className":10412},[126],[29,10414,10416],{"className":10415,"style":161},[130],[29,10417],{}," 就取代了 ",[29,10420,10422,10440],{"className":10421},[32],[29,10423,10425],{"className":10424},[36],[38,10426,10427],{"xmlns":40},[42,10428,10429,10437],{},[45,10430,10431],{},[48,10432,10433,10435],{},[51,10434,1045],{},[51,10436,296],{},[89,10438,10439],{"encoding":91},"k_i",[29,10441,10443],{"className":10442,"ariaHidden":61},[96],[29,10444,10446,10449],{"className":10445},[100],[29,10447],{"className":10448,"style":3098},[104],[29,10450,10452,10455],{"className":10451},[109],[29,10453,1045],{"className":10454,"style":1387},[109,113],[29,10456,10458],{"className":10457},[117],[29,10459,10461,10481],{"className":10460},[121,122],[29,10462,10464,10478],{"className":10463},[126],[29,10465,10467],{"className":10466,"style":341},[130],[29,10468,10469,10472],{"style":9323},[29,10470],{"className":10471,"style":139},[138],[29,10473,10475],{"className":10474},[143,144,145,146],[29,10476,296],{"className":10477},[109,113,146],[29,10479,154],{"className":10480},[153],[29,10482,10484],{"className":10483},[126],[29,10485,10487],{"className":10486,"style":161},[130],[29,10488],{},"。同理 ",[29,10491,10493,10541],{"className":10492},[32],[29,10494,10496],{"className":10495},[36],[38,10497,10498],{"xmlns":40},[42,10499,10500,10538],{},[45,10501,10502,10516,10518,10524],{},[625,10503,10504,10506,10508],{},[51,10505,1067],{},[51,10507,296],{},[45,10509,10510,10512,10514],{},[59,10511,460],{"stretchy":459},[51,10513,463],{},[59,10515,466],{"stretchy":459},[59,10517,508],{},[48,10519,10520,10522],{},[51,10521,6467],{},[51,10523,296],{},[625,10525,10526,10528,10530],{},[51,10527,2087],{},[51,10529,1067],{},[45,10531,10532,10534,10536],{},[59,10533,460],{"stretchy":459},[51,10535,463],{},[59,10537,466],{"stretchy":459},[89,10539,10540],{"encoding":91},"v_i^{(s)} = c_i W_v^{(s)}",[29,10542,10544,10619],{"className":10543,"ariaHidden":61},[96],[29,10545,10547,10550,10610,10613,10616],{"className":10546},[100],[29,10548],{"className":10549,"style":2227},[104],[29,10551,10553,10556],{"className":10552},[109],[29,10554,1067],{"className":10555,"style":1319},[109,113],[29,10557,10559],{"className":10558},[117],[29,10560,10562,10602],{"className":10561},[121,122],[29,10563,10565,10599],{"className":10564},[126],[29,10566,10568,10579],{"className":10567,"style":771},[130],[29,10569,10570,10573],{"style":1990},[29,10571],{"className":10572,"style":139},[138],[29,10574,10576],{"className":10575},[143,144,145,146],[29,10577,296],{"className":10578},[109,113,146],[29,10580,10581,10584],{"style":786},[29,10582],{"className":10583,"style":139},[138],[29,10585,10587],{"className":10586},[143,144,145,146],[29,10588,10590,10593,10596],{"className":10589},[109,146],[29,10591,460],{"className":10592},[483,146],[29,10594,463],{"className":10595},[109,113,146],[29,10597,466],{"className":10598},[490,146],[29,10600,154],{"className":10601},[153],[29,10603,10605],{"className":10604},[126],[29,10606,10608],{"className":10607,"style":1760},[130],[29,10609],{},[29,10611],{"className":10612,"style":367},[171],[29,10614,508],{"className":10615},[371],[29,10617],{"className":10618,"style":367},[171],[29,10620,10622,10626,10666],{"className":10621},[100],[29,10623],{"className":10624,"style":10625},[104],"height:1.1948em;vertical-align:-0.15em;",[29,10627,10629,10632],{"className":10628},[109],[29,10630,6467],{"className":10631},[109,113],[29,10633,10635],{"className":10634},[117],[29,10636,10638,10658],{"className":10637},[121,122],[29,10639,10641,10655],{"className":10640},[126],[29,10642,10644],{"className":10643,"style":341},[130],[29,10645,10646,10649],{"style":134},[29,10647],{"className":10648,"style":139},[138],[29,10650,10652],{"className":10651},[143,144,145,146],[29,10653,296],{"className":10654},[109,113,146],[29,10656,154],{"className":10657},[153],[29,10659,10661],{"className":10660},[126],[29,10662,10664],{"className":10663,"style":161},[130],[29,10665],{},[29,10667,10669,10672],{"className":10668},[109],[29,10670,2087],{"className":10671,"style":2350},[109,113],[29,10673,10675],{"className":10674},[117],[29,10676,10678,10718],{"className":10677},[121,122],[29,10679,10681,10715],{"className":10680},[126],[29,10682,10684,10695],{"className":10683,"style":771},[130],[29,10685,10686,10689],{"style":10243},[29,10687],{"className":10688,"style":139},[138],[29,10690,10692],{"className":10691},[143,144,145,146],[29,10693,1067],{"className":10694,"style":1319},[109,113,146],[29,10696,10697,10700],{"style":786},[29,10698],{"className":10699,"style":139},[138],[29,10701,10703],{"className":10702},[143,144,145,146],[29,10704,10706,10709,10712],{"className":10705},[109,146],[29,10707,460],{"className":10708},[483,146],[29,10710,463],{"className":10711},[109,113,146],[29,10713,466],{"className":10714},[490,146],[29,10716,154],{"className":10717},[153],[29,10719,10721],{"className":10720},[126],[29,10722,10725],{"className":10723,"style":10724},[130],"height:0.1166em;",[29,10726],{}," 里的 ",[29,10729,10731,10757],{"className":10730},[32],[29,10732,10734],{"className":10733},[36],[38,10735,10736],{"xmlns":40},[42,10737,10738,10754],{},[45,10739,10740],{},[625,10741,10742,10744,10746],{},[51,10743,2087],{},[51,10745,1067],{},[45,10747,10748,10750,10752],{},[59,10749,460],{"stretchy":459},[51,10751,463],{},[59,10753,466],{"stretchy":459},[89,10755,10756],{"encoding":91},"W_v^{(s)}",[29,10758,10760],{"className":10759,"ariaHidden":61},[96],[29,10761,10763,10767],{"className":10762},[100],[29,10764],{"className":10765,"style":10766},[104],"height:1.1614em;vertical-align:-0.1166em;",[29,10768,10770,10773],{"className":10769},[109],[29,10771,2087],{"className":10772,"style":2350},[109,113],[29,10774,10776],{"className":10775},[117],[29,10777,10779,10819],{"className":10778},[121,122],[29,10780,10782,10816],{"className":10781},[126],[29,10783,10785,10796],{"className":10784,"style":771},[130],[29,10786,10787,10790],{"style":10243},[29,10788],{"className":10789,"style":139},[138],[29,10791,10793],{"className":10792},[143,144,145,146],[29,10794,1067],{"className":10795,"style":1319},[109,113,146],[29,10797,10798,10801],{"style":786},[29,10799],{"className":10800,"style":139},[138],[29,10802,10804],{"className":10803},[143,144,145,146],[29,10805,10807,10810,10813],{"className":10806},[109,146],[29,10808,460],{"className":10809},[483,146],[29,10811,463],{"className":10812},[109,113,146],[29,10814,466],{"className":10815},[490,146],[29,10817,154],{"className":10818},[153],[29,10820,10822],{"className":10821},[126],[29,10823,10825],{"className":10824,"style":10724},[130],[29,10826],{}," 也能吸收进后面的输出投影，",[29,10829,10831,10849],{"className":10830},[32],[29,10832,10834],{"className":10833},[36],[38,10835,10836],{"xmlns":40},[42,10837,10838,10846],{},[45,10839,10840],{},[48,10841,10842,10844],{},[51,10843,1067],{},[51,10845,296],{},[89,10847,10848],{"encoding":91},"v_i",[29,10850,10852],{"className":10851,"ariaHidden":61},[96],[29,10853,10855,10858],{"className":10854},[100],[29,10856],{"className":10857,"style":692},[104],[29,10859,10861,10864],{"className":10860},[109],[29,10862,1067],{"className":10863,"style":1319},[109,113],[29,10865,10867],{"className":10866},[117],[29,10868,10870,10890],{"className":10869},[121,122],[29,10871,10873,10887],{"className":10872},[126],[29,10874,10876],{"className":10875,"style":341},[130],[29,10877,10878,10881],{"style":9370},[29,10879],{"className":10880,"style":139},[138],[29,10882,10884],{"className":10883},[143,144,145,146],[29,10885,296],{"className":10886},[109,113,146],[29,10888,154],{"className":10889},[153],[29,10891,10893],{"className":10892},[126],[29,10894,10896],{"className":10895,"style":161},[130],[29,10897],{}," 也可以用 ",[29,10900,10902,10919],{"className":10901},[32],[29,10903,10905],{"className":10904},[36],[38,10906,10907],{"xmlns":40},[42,10908,10909,10917],{},[45,10910,10911],{},[48,10912,10913,10915],{},[51,10914,6467],{},[51,10916,296],{},[89,10918,6472],{"encoding":91},[29,10920,10922],{"className":10921,"ariaHidden":61},[96],[29,10923,10925,10928],{"className":10924},[100],[29,10926],{"className":10927,"style":692},[104],[29,10929,10931,10934],{"className":10930},[109],[29,10932,6467],{"className":10933},[109,113],[29,10935,10937],{"className":10936},[117],[29,10938,10940,10960],{"className":10939},[121,122],[29,10941,10943,10957],{"className":10942},[126],[29,10944,10946],{"className":10945,"style":341},[130],[29,10947,10948,10951],{"style":134},[29,10949],{"className":10950,"style":139},[138],[29,10952,10954],{"className":10953},[143,144,145,146],[29,10955,296],{"className":10956},[109,113,146],[29,10958,154],{"className":10959},[153],[29,10961,10963],{"className":10962},[126],[29,10964,10966],{"className":10965,"style":161},[130],[29,10967],{}," 代替。于是推理时 KV Cache 只需要存 ",[29,10970,10972,10989],{"className":10971},[32],[29,10973,10975],{"className":10974},[36],[38,10976,10977],{"xmlns":40},[42,10978,10979,10987],{},[45,10980,10981],{},[48,10982,10983,10985],{},[51,10984,6467],{},[51,10986,296],{},[89,10988,6472],{"encoding":91},[29,10990,10992],{"className":10991,"ariaHidden":61},[96],[29,10993,10995,10998],{"className":10994},[100],[29,10996],{"className":10997,"style":692},[104],[29,10999,11001,11004],{"className":11000},[109],[29,11002,6467],{"className":11003},[109,113],[29,11005,11007],{"className":11006},[117],[29,11008,11010,11030],{"className":11009},[121,122],[29,11011,11013,11027],{"className":11012},[126],[29,11014,11016],{"className":11015,"style":341},[130],[29,11017,11018,11021],{"style":134},[29,11019],{"className":11020,"style":139},[138],[29,11022,11024],{"className":11023},[143,144,145,146],[29,11025,296],{"className":11026},[109,113,146],[29,11028,154],{"className":11029},[153],[29,11031,11033],{"className":11032},[126],[29,11034,11036],{"className":11035,"style":161},[130],[29,11037],{},"。而 ",[29,11040,11042,11059],{"className":11041},[32],[29,11043,11045],{"className":11044},[36],[38,11046,11047],{"xmlns":40},[42,11048,11049,11057],{},[45,11050,11051],{},[48,11052,11053,11055],{},[51,11054,6467],{},[51,11056,296],{},[89,11058,6472],{"encoding":91},[29,11060,11062],{"className":11061,"ariaHidden":61},[96],[29,11063,11065,11068],{"className":11064},[100],[29,11066],{"className":11067,"style":692},[104],[29,11069,11071,11074],{"className":11070},[109],[29,11072,6467],{"className":11073},[109,113],[29,11075,11077],{"className":11076},[117],[29,11078,11080,11100],{"className":11079},[121,122],[29,11081,11083,11097],{"className":11082},[126],[29,11084,11086],{"className":11085,"style":341},[130],[29,11087,11088,11091],{"style":134},[29,11089],{"className":11090,"style":139},[138],[29,11092,11094],{"className":11093},[143,144,145,146],[29,11095,296],{"className":11096},[109,113,146],[29,11098,154],{"className":11099},[153],[29,11101,11103],{"className":11102},[126],[29,11104,11106],{"className":11105,"style":161},[130],[29,11107],{}," 与 ",[29,11110,11112,11129],{"className":11111},[32],[29,11113,11115],{"className":11114},[36],[38,11116,11117],{"xmlns":40},[42,11118,11119,11127],{},[45,11120,11121,11123,11125],{},[59,11122,460],{"stretchy":459},[51,11124,463],{},[59,11126,466],{"stretchy":459},[89,11128,469],{"encoding":91},[29,11130,11132],{"className":11131,"ariaHidden":61},[96],[29,11133,11135,11138,11141,11144],{"className":11134},[100],[29,11136],{"className":11137,"style":479},[104],[29,11139,460],{"className":11140},[483],[29,11142,463],{"className":11143},[109,113],[29,11145,466],{"className":11146},[490]," 无关，是所有 head 共享的。",[10,11149,11150,11153,11154,11224,11225,11373],{},[14,11151,11152],{},"MLA 在推理阶段恒等变换成了一个 MQA。"," 到这里 MLA 完成了什么就很清楚了：它用不同的投影矩阵增强了 GQA 的能力，同时在推理时保持同样大小的 KV Cache。反过来想，如果只需要跟 GQA 相当的能力，那 ",[29,11155,11157,11175],{"className":11156},[32],[29,11158,11160],{"className":11159},[36],[38,11161,11162],{"xmlns":40},[42,11163,11164,11172],{},[45,11165,11166],{},[48,11167,11168,11170],{},[51,11169,309],{},[51,11171,6467],{},[89,11173,11174],{"encoding":91},"d_c",[29,11176,11178],{"className":11177,"ariaHidden":61},[96],[29,11179,11181,11184],{"className":11180},[100],[29,11182],{"className":11183,"style":3098},[104],[29,11185,11187,11190],{"className":11186},[109],[29,11188,309],{"className":11189},[109,113],[29,11191,11193],{"className":11192},[117],[29,11194,11196,11216],{"className":11195},[121,122],[29,11197,11199,11213],{"className":11198},[126],[29,11200,11202],{"className":11201,"style":3172},[130],[29,11203,11204,11207],{"style":134},[29,11205],{"className":11206,"style":139},[138],[29,11208,11210],{"className":11209},[143,144,145,146],[29,11211,6467],{"className":11212},[109,113,146],[29,11214,154],{"className":11215},[153],[29,11217,11219],{"className":11218},[126],[29,11220,11222],{"className":11221,"style":161},[130],[29,11223],{}," 就不必取 ",[29,11226,11228,11260],{"className":11227},[32],[29,11229,11231],{"className":11230},[36],[38,11232,11233],{"xmlns":40},[42,11234,11235,11257],{},[45,11236,11237,11239,11241,11247,11249,11255],{},[51,11238,5201],{},[59,11240,460],{"stretchy":459},[48,11242,11243,11245],{},[51,11244,309],{},[51,11246,1045],{},[59,11248,3236],{},[48,11250,11251,11253],{},[51,11252,309],{},[51,11254,1067],{},[59,11256,466],{"stretchy":459},[89,11258,11259],{"encoding":91},"g(d_k + d_v)",[29,11261,11263,11324],{"className":11262,"ariaHidden":61},[96],[29,11264,11266,11269,11272,11275,11315,11318,11321],{"className":11265},[100],[29,11267],{"className":11268,"style":479},[104],[29,11270,5201],{"className":11271,"style":1319},[109,113],[29,11273,460],{"className":11274},[483],[29,11276,11278,11281],{"className":11277},[109],[29,11279,309],{"className":11280},[109,113],[29,11282,11284],{"className":11283},[117],[29,11285,11287,11307],{"className":11286},[121,122],[29,11288,11290,11304],{"className":11289},[126],[29,11291,11293],{"className":11292,"style":253},[130],[29,11294,11295,11298],{"style":134},[29,11296],{"className":11297,"style":139},[138],[29,11299,11301],{"className":11300},[143,144,145,146],[29,11302,1045],{"className":11303,"style":1387},[109,113,146],[29,11305,154],{"className":11306},[153],[29,11308,11310],{"className":11309},[126],[29,11311,11313],{"className":11312,"style":161},[130],[29,11314],{},[29,11316],{"className":11317,"style":3258},[171],[29,11319,3236],{"className":11320},[3262],[29,11322],{"className":11323,"style":3258},[171],[29,11325,11327,11330,11370],{"className":11326},[100],[29,11328],{"className":11329,"style":479},[104],[29,11331,11333,11336],{"className":11332},[109],[29,11334,309],{"className":11335},[109,113],[29,11337,11339],{"className":11338},[117],[29,11340,11342,11362],{"className":11341},[121,122],[29,11343,11345,11359],{"className":11344},[126],[29,11346,11348],{"className":11347,"style":3172},[130],[29,11349,11350,11353],{"style":134},[29,11351],{"className":11352,"style":139},[138],[29,11354,11356],{"className":11355},[143,144,145,146],[29,11357,1067],{"className":11358,"style":1319},[109,113,146],[29,11360,154],{"className":11361},[153],[29,11363,11365],{"className":11364},[126],[29,11366,11368],{"className":11367,"style":161},[130],[29,11369],{},[29,11371,466],{"className":11372},[490],"，取更小的值就行。DeepSeek-V2 取了 512。",[10,11375,11376],{},"这就是 MLA 的核心。低秩压缩为什么有效？因为不同 head 需要的信息本来就有大量重叠。下投影矩阵负责提取这些共同信息，上投影矩阵让各个 head 用各自的方式去使用它。这甚至可能带来一点正则化效果。它强制模型把冗余的共同信息压进同一个低维空间，而不是每个 head 各存一份。",[10,11378,11379],{},"我一开始以为 MLA 只是\"把 KV 存小一点\"。后来才反应过来，它换掉的是一整套共享方式。存小只是结果，不是原因。",[3046,11381,11383,11562],{"title":11382},"数值精度",[10,11384,11385,11561],{},[29,11386,11388,11429],{"className":11387},[32],[29,11389,11391],{"className":11390},[36],[38,11392,11393],{"xmlns":40},[42,11394,11395,11427],{},[45,11396,11397,11411],{},[625,11398,11399,11401,11403],{},[51,11400,2087],{},[51,11402,1028],{},[45,11404,11405,11407,11409],{},[59,11406,460],{"stretchy":459},[51,11408,463],{},[59,11410,466],{"stretchy":459},[625,11412,11413,11415,11417],{},[51,11414,2087],{},[51,11416,1045],{},[45,11418,11419,11421,11423,11425],{},[59,11420,460],{"stretchy":459},[51,11422,463],{},[59,11424,466],{"stretchy":459},[51,11426,1144],{"mathvariant":1002},[89,11428,10213],{"encoding":91},[29,11430,11432],{"className":11431,"ariaHidden":61},[96],[29,11433,11435,11438,11498],{"className":11434},[100],[29,11436],{"className":11437,"style":2585},[104],[29,11439,11441,11444],{"className":11440},[109],[29,11442,2087],{"className":11443,"style":2350},[109,113],[29,11445,11447],{"className":11446},[117],[29,11448,11450,11490],{"className":11449},[121,122],[29,11451,11453,11487],{"className":11452},[126],[29,11454,11456,11467],{"className":11455,"style":771},[130],[29,11457,11458,11461],{"style":10243},[29,11459],{"className":11460,"style":139},[138],[29,11462,11464],{"className":11463},[143,144,145,146],[29,11465,1028],{"className":11466,"style":1319},[109,113,146],[29,11468,11469,11472],{"style":786},[29,11470],{"className":11471,"style":139},[138],[29,11473,11475],{"className":11474},[143,144,145,146],[29,11476,11478,11481,11484],{"className":11477},[109,146],[29,11479,460],{"className":11480},[483,146],[29,11482,463],{"className":11483},[109,113,146],[29,11485,466],{"className":11486},[490,146],[29,11488,154],{"className":11489},[153],[29,11491,11493],{"className":11492},[126],[29,11494,11496],{"className":11495,"style":10282},[130],[29,11497],{},[29,11499,11501,11504],{"className":11500},[109],[29,11502,2087],{"className":11503,"style":2350},[109,113],[29,11505,11507],{"className":11506},[117],[29,11508,11510,11553],{"className":11509},[121,122],[29,11511,11513,11550],{"className":11512},[126],[29,11514,11516,11527],{"className":11515,"style":771},[130],[29,11517,11518,11521],{"style":2646},[29,11519],{"className":11520,"style":139},[138],[29,11522,11524],{"className":11523},[143,144,145,146],[29,11525,1045],{"className":11526,"style":1387},[109,113,146],[29,11528,11529,11532],{"style":786},[29,11530],{"className":11531,"style":139},[138],[29,11533,11535],{"className":11534},[143,144,145,146],[29,11536,11538,11541,11544,11547],{"className":11537},[109,146],[29,11539,460],{"className":11540},[483,146],[29,11542,463],{"className":11543},[109,113,146],[29,11545,466],{"className":11546},[490,146],[29,11548,1144],{"className":11549},[109,146],[29,11551,154],{"className":11552},[153],[29,11554,11556],{"className":11555},[126],[29,11557,11559],{"className":11558,"style":2685},[130],[29,11560],{}," 合并成一个矩阵，理论上只在无限精度下成立。用 BF16 时变换后的精度损失往往挺明显，多层累积后可能放大到可观的程度。",[10,11563,11564,11565,11798,11799,12032],{},"实际实现一般不按 ",[29,11566,11568,11620],{"className":11567},[32],[29,11569,11571],{"className":11570},[36],[38,11572,11573],{"xmlns":40},[42,11574,11575,11617],{},[45,11576,11577,11583,11585,11599,11615],{},[48,11578,11579,11581],{},[51,11580,53],{},[51,11582,618],{},[59,11584,460],{"stretchy":459},[625,11586,11587,11589,11591],{},[51,11588,2087],{},[51,11590,1028],{},[45,11592,11593,11595,11597],{},[59,11594,460],{"stretchy":459},[51,11596,463],{},[59,11598,466],{"stretchy":459},[625,11600,11601,11603,11605],{},[51,11602,2087],{},[51,11604,1045],{},[45,11606,11607,11609,11611,11613],{},[59,11608,460],{"stretchy":459},[51,11610,463],{},[59,11612,466],{"stretchy":459},[51,11614,1144],{"mathvariant":1002},[59,11616,466],{"stretchy":459},[89,11618,11619],{"encoding":91},"x_t (W_q^{(s)} W_k^{(s)\\top})",[29,11621,11623],{"className":11622,"ariaHidden":61},[96],[29,11624,11626,11629,11669,11672,11732,11795],{"className":11625},[100],[29,11627],{"className":11628,"style":2585},[104],[29,11630,11632,11635],{"className":11631},[109],[29,11633,53],{"className":11634},[109,113],[29,11636,11638],{"className":11637},[117],[29,11639,11641,11661],{"className":11640},[121,122],[29,11642,11644,11658],{"className":11643},[126],[29,11645,11647],{"className":11646,"style":711},[130],[29,11648,11649,11652],{"style":134},[29,11650],{"className":11651,"style":139},[138],[29,11653,11655],{"className":11654},[143,144,145,146],[29,11656,618],{"className":11657},[109,113,146],[29,11659,154],{"className":11660},[153],[29,11662,11664],{"className":11663},[126],[29,11665,11667],{"className":11666,"style":161},[130],[29,11668],{},[29,11670,460],{"className":11671},[483],[29,11673,11675,11678],{"className":11674},[109],[29,11676,2087],{"className":11677,"style":2350},[109,113],[29,11679,11681],{"className":11680},[117],[29,11682,11684,11724],{"className":11683},[121,122],[29,11685,11687,11721],{"className":11686},[126],[29,11688,11690,11701],{"className":11689,"style":771},[130],[29,11691,11692,11695],{"style":10243},[29,11693],{"className":11694,"style":139},[138],[29,11696,11698],{"className":11697},[143,144,145,146],[29,11699,1028],{"className":11700,"style":1319},[109,113,146],[29,11702,11703,11706],{"style":786},[29,11704],{"className":11705,"style":139},[138],[29,11707,11709],{"className":11708},[143,144,145,146],[29,11710,11712,11715,11718],{"className":11711},[109,146],[29,11713,460],{"className":11714},[483,146],[29,11716,463],{"className":11717},[109,113,146],[29,11719,466],{"className":11720},[490,146],[29,11722,154],{"className":11723},[153],[29,11725,11727],{"className":11726},[126],[29,11728,11730],{"className":11729,"style":10282},[130],[29,11731],{},[29,11733,11735,11738],{"className":11734},[109],[29,11736,2087],{"className":11737,"style":2350},[109,113],[29,11739,11741],{"className":11740},[117],[29,11742,11744,11787],{"className":11743},[121,122],[29,11745,11747,11784],{"className":11746},[126],[29,11748,11750,11761],{"className":11749,"style":771},[130],[29,11751,11752,11755],{"style":2646},[29,11753],{"className":11754,"style":139},[138],[29,11756,11758],{"className":11757},[143,144,145,146],[29,11759,1045],{"className":11760,"style":1387},[109,113,146],[29,11762,11763,11766],{"style":786},[29,11764],{"className":11765,"style":139},[138],[29,11767,11769],{"className":11768},[143,144,145,146],[29,11770,11772,11775,11778,11781],{"className":11771},[109,146],[29,11773,460],{"className":11774},[483,146],[29,11776,463],{"className":11777},[109,113,146],[29,11779,466],{"className":11780},[490,146],[29,11782,1144],{"className":11783},[109,146],[29,11785,154],{"className":11786},[153],[29,11788,11790],{"className":11789},[126],[29,11791,11793],{"className":11792,"style":2685},[130],[29,11794],{},[29,11796,466],{"className":11797},[490]," 算 Q，而是按 ",[29,11800,11802,11854],{"className":11801},[32],[29,11803,11805],{"className":11804},[36],[38,11806,11807],{"xmlns":40},[42,11808,11809,11851],{},[45,11810,11811,11813,11819,11833,11835],{},[59,11812,460],{"stretchy":459},[48,11814,11815,11817],{},[51,11816,53],{},[51,11818,618],{},[625,11820,11821,11823,11825],{},[51,11822,2087],{},[51,11824,1028],{},[45,11826,11827,11829,11831],{},[59,11828,460],{"stretchy":459},[51,11830,463],{},[59,11832,466],{"stretchy":459},[59,11834,466],{"stretchy":459},[625,11836,11837,11839,11841],{},[51,11838,2087],{},[51,11840,1045],{},[45,11842,11843,11845,11847,11849],{},[59,11844,460],{"stretchy":459},[51,11846,463],{},[59,11848,466],{"stretchy":459},[51,11850,1144],{"mathvariant":1002},[89,11852,11853],{"encoding":91},"(x_t W_q^{(s)}) W_k^{(s)\\top}",[29,11855,11857],{"className":11856,"ariaHidden":61},[96],[29,11858,11860,11863,11866,11906,11966,11969],{"className":11859},[100],[29,11861],{"className":11862,"style":2585},[104],[29,11864,460],{"className":11865},[483],[29,11867,11869,11872],{"className":11868},[109],[29,11870,53],{"className":11871},[109,113],[29,11873,11875],{"className":11874},[117],[29,11876,11878,11898],{"className":11877},[121,122],[29,11879,11881,11895],{"className":11880},[126],[29,11882,11884],{"className":11883,"style":711},[130],[29,11885,11886,11889],{"style":134},[29,11887],{"className":11888,"style":139},[138],[29,11890,11892],{"className":11891},[143,144,145,146],[29,11893,618],{"className":11894},[109,113,146],[29,11896,154],{"className":11897},[153],[29,11899,11901],{"className":11900},[126],[29,11902,11904],{"className":11903,"style":161},[130],[29,11905],{},[29,11907,11909,11912],{"className":11908},[109],[29,11910,2087],{"className":11911,"style":2350},[109,113],[29,11913,11915],{"className":11914},[117],[29,11916,11918,11958],{"className":11917},[121,122],[29,11919,11921,11955],{"className":11920},[126],[29,11922,11924,11935],{"className":11923,"style":771},[130],[29,11925,11926,11929],{"style":10243},[29,11927],{"className":11928,"style":139},[138],[29,11930,11932],{"className":11931},[143,144,145,146],[29,11933,1028],{"className":11934,"style":1319},[109,113,146],[29,11936,11937,11940],{"style":786},[29,11938],{"className":11939,"style":139},[138],[29,11941,11943],{"className":11942},[143,144,145,146],[29,11944,11946,11949,11952],{"className":11945},[109,146],[29,11947,460],{"className":11948},[483,146],[29,11950,463],{"className":11951},[109,113,146],[29,11953,466],{"className":11954},[490,146],[29,11956,154],{"className":11957},[153],[29,11959,11961],{"className":11960},[126],[29,11962,11964],{"className":11963,"style":10282},[130],[29,11965],{},[29,11967,466],{"className":11968},[490],[29,11970,11972,11975],{"className":11971},[109],[29,11973,2087],{"className":11974,"style":2350},[109,113],[29,11976,11978],{"className":11977},[117],[29,11979,11981,12024],{"className":11980},[121,122],[29,11982,11984,12021],{"className":11983},[126],[29,11985,11987,11998],{"className":11986,"style":771},[130],[29,11988,11989,11992],{"style":2646},[29,11990],{"className":11991,"style":139},[138],[29,11993,11995],{"className":11994},[143,144,145,146],[29,11996,1045],{"className":11997,"style":1387},[109,113,146],[29,11999,12000,12003],{"style":786},[29,12001],{"className":12002,"style":139},[138],[29,12004,12006],{"className":12005},[143,144,145,146],[29,12007,12009,12012,12015,12018],{"className":12008},[109,146],[29,12010,460],{"className":12011},[483,146],[29,12013,463],{"className":12014},[109,113,146],[29,12016,466],{"className":12017},[490,146],[29,12019,1144],{"className":12020},[109,146],[29,12022,154],{"className":12023},[153],[29,12025,12027],{"className":12026},[126],[29,12028,12030],{"className":12029,"style":2685},[130],[29,12031],{}," 算。虽然是串行的，但在低秩假设下计算量更少，理论精度损失也更小。",[10,12034,12035,12036,12039,12040,12216],{},"不过别急。到这里 MLA 有个绕不开的缺陷：",[14,12037,12038],{},"不兼容 RoPE","。前面那步恒等变换能成立，关键在于 ",[29,12041,12043,12084],{"className":12042},[32],[29,12044,12046],{"className":12045},[36],[38,12047,12048],{"xmlns":40},[42,12049,12050,12082],{},[45,12051,12052,12066],{},[625,12053,12054,12056,12058],{},[51,12055,2087],{},[51,12057,1028],{},[45,12059,12060,12062,12064],{},[59,12061,460],{"stretchy":459},[51,12063,463],{},[59,12065,466],{"stretchy":459},[625,12067,12068,12070,12072],{},[51,12069,2087],{},[51,12071,1045],{},[45,12073,12074,12076,12078,12080],{},[59,12075,460],{"stretchy":459},[51,12077,463],{},[59,12079,466],{"stretchy":459},[51,12081,1144],{"mathvariant":1002},[89,12083,10213],{"encoding":91},[29,12085,12087],{"className":12086,"ariaHidden":61},[96],[29,12088,12090,12093,12153],{"className":12089},[100],[29,12091],{"className":12092,"style":2585},[104],[29,12094,12096,12099],{"className":12095},[109],[29,12097,2087],{"className":12098,"style":2350},[109,113],[29,12100,12102],{"className":12101},[117],[29,12103,12105,12145],{"className":12104},[121,122],[29,12106,12108,12142],{"className":12107},[126],[29,12109,12111,12122],{"className":12110,"style":771},[130],[29,12112,12113,12116],{"style":10243},[29,12114],{"className":12115,"style":139},[138],[29,12117,12119],{"className":12118},[143,144,145,146],[29,12120,1028],{"className":12121,"style":1319},[109,113,146],[29,12123,12124,12127],{"style":786},[29,12125],{"className":12126,"style":139},[138],[29,12128,12130],{"className":12129},[143,144,145,146],[29,12131,12133,12136,12139],{"className":12132},[109,146],[29,12134,460],{"className":12135},[483,146],[29,12137,463],{"className":12138},[109,113,146],[29,12140,466],{"className":12141},[490,146],[29,12143,154],{"className":12144},[153],[29,12146,12148],{"className":12147},[126],[29,12149,12151],{"className":12150,"style":10282},[130],[29,12152],{},[29,12154,12156,12159],{"className":12155},[109],[29,12157,2087],{"className":12158,"style":2350},[109,113],[29,12160,12162],{"className":12161},[117],[29,12163,12165,12208],{"className":12164},[121,122],[29,12166,12168,12205],{"className":12167},[126],[29,12169,12171,12182],{"className":12170,"style":771},[130],[29,12172,12173,12176],{"style":2646},[29,12174],{"className":12175,"style":139},[138],[29,12177,12179],{"className":12178},[143,144,145,146],[29,12180,1045],{"className":12181,"style":1387},[109,113,146],[29,12183,12184,12187],{"style":786},[29,12185],{"className":12186,"style":139},[138],[29,12188,12190],{"className":12189},[143,144,145,146],[29,12191,12193,12196,12199,12202],{"className":12192},[109,146],[29,12194,460],{"className":12195},[483,146],[29,12197,463],{"className":12198},[109,113,146],[29,12200,466],{"className":12201},[490,146],[29,12203,1144],{"className":12204},[109,146],[29,12206,154],{"className":12207},[153],[29,12209,12211],{"className":12210},[126],[29,12212,12214],{"className":12213,"style":2685},[130],[29,12215],{}," 是一个跟位置无关的固定矩阵。加上 RoPE 之后就不行了：",[29,12218,12220],{"className":12219},[597],[29,12221,12223,12394],{"className":12222},[32],[29,12224,12226],{"className":12225},[36],[38,12227,12228],{"xmlns":40,"display":606},[42,12229,12230,12391],{},[45,12231,12232,12246,12262,12264,12266,12272,12286,12292,12294,12296,12302,12316,12322,12328,12330,12336,12338,12352,12365,12381,12383],{},[625,12233,12234,12236,12238],{},[51,12235,1028],{},[51,12237,618],{},[45,12239,12240,12242,12244],{},[59,12241,460],{"stretchy":459},[51,12243,463],{},[59,12245,466],{"stretchy":459},[625,12247,12248,12250,12252],{},[51,12249,1045],{},[51,12251,296],{},[45,12253,12254,12256,12258,12260],{},[59,12255,460],{"stretchy":459},[51,12257,463],{},[59,12259,466],{"stretchy":459},[51,12261,1144],{"mathvariant":1002},[59,12263,508],{},[59,12265,460],{"stretchy":459},[48,12267,12268,12270],{},[51,12269,53],{},[51,12271,618],{},[625,12273,12274,12276,12278],{},[51,12275,2087],{},[51,12277,1028],{},[45,12279,12280,12282,12284],{},[59,12281,460],{"stretchy":459},[51,12283,463],{},[59,12285,466],{"stretchy":459},[48,12287,12288,12290],{},[51,12289,306],{},[51,12291,618],{},[59,12293,466],{"stretchy":459},[59,12295,460],{"stretchy":459},[48,12297,12298,12300],{},[51,12299,6467],{},[51,12301,296],{},[625,12303,12304,12306,12308],{},[51,12305,2087],{},[51,12307,1045],{},[45,12309,12310,12312,12314],{},[59,12311,460],{"stretchy":459},[51,12313,463],{},[59,12315,466],{"stretchy":459},[48,12317,12318,12320],{},[51,12319,306],{},[51,12321,296],{},[301,12323,12324,12326],{},[59,12325,466],{"stretchy":459},[51,12327,1144],{"mathvariant":1002},[59,12329,508],{},[48,12331,12332,12334],{},[51,12333,53],{},[51,12335,618],{},[59,12337,460],{"stretchy":459},[625,12339,12340,12342,12344],{},[51,12341,2087],{},[51,12343,1028],{},[45,12345,12346,12348,12350],{},[59,12347,460],{"stretchy":459},[51,12349,463],{},[59,12351,466],{"stretchy":459},[48,12353,12354,12356],{},[51,12355,306],{},[45,12357,12358,12360,12363],{},[51,12359,618],{},[59,12361,12362],{},"−",[51,12364,296],{},[625,12366,12367,12369,12371],{},[51,12368,2087],{},[51,12370,1045],{},[45,12372,12373,12375,12377,12379],{},[59,12374,460],{"stretchy":459},[51,12376,463],{},[59,12378,466],{"stretchy":459},[51,12380,1144],{"mathvariant":1002},[59,12382,466],{"stretchy":459},[625,12384,12385,12387,12389],{},[51,12386,6467],{},[51,12388,296],{},[51,12390,1144],{"mathvariant":1002},[89,12392,12393],{"encoding":91},"q_t^{(s)} k_i^{(s)\\top} = (x_t W_q^{(s)} R_t)(c_i W_k^{(s)} R_i)^\\top = x_t (W_q^{(s)} R_{t-i} W_k^{(s)\\top}) c_i^\\top",[29,12395,12397,12535,12870],{"className":12396,"ariaHidden":61},[96],[29,12398,12400,12403,12463,12526,12529,12532],{"className":12399},[100],[29,12401],{"className":12402,"style":2227},[104],[29,12404,12406,12409],{"className":12405},[109],[29,12407,1028],{"className":12408,"style":1319},[109,113],[29,12410,12412],{"className":12411},[117],[29,12413,12415,12455],{"className":12414},[121,122],[29,12416,12418,12452],{"className":12417},[126],[29,12419,12421,12432],{"className":12420,"style":771},[130],[29,12422,12423,12426],{"style":1334},[29,12424],{"className":12425,"style":139},[138],[29,12427,12429],{"className":12428},[143,144,145,146],[29,12430,618],{"className":12431},[109,113,146],[29,12433,12434,12437],{"style":786},[29,12435],{"className":12436,"style":139},[138],[29,12438,12440],{"className":12439},[143,144,145,146],[29,12441,12443,12446,12449],{"className":12442},[109,146],[29,12444,460],{"className":12445},[483,146],[29,12447,463],{"className":12448},[109,113,146],[29,12450,466],{"className":12451},[490,146],[29,12453,154],{"className":12454},[153],[29,12456,12458],{"className":12457},[126],[29,12459,12461],{"className":12460,"style":814},[130],[29,12462],{},[29,12464,12466,12469],{"className":12465},[109],[29,12467,1045],{"className":12468,"style":1387},[109,113],[29,12470,12472],{"className":12471},[117],[29,12473,12475,12518],{"className":12474},[121,122],[29,12476,12478,12515],{"className":12477},[126],[29,12479,12481,12492],{"className":12480,"style":771},[130],[29,12482,12483,12486],{"style":1718},[29,12484],{"className":12485,"style":139},[138],[29,12487,12489],{"className":12488},[143,144,145,146],[29,12490,296],{"className":12491},[109,113,146],[29,12493,12494,12497],{"style":786},[29,12495],{"className":12496,"style":139},[138],[29,12498,12500],{"className":12499},[143,144,145,146],[29,12501,12503,12506,12509,12512],{"className":12502},[109,146],[29,12504,460],{"className":12505},[483,146],[29,12507,463],{"className":12508},[109,113,146],[29,12510,466],{"className":12511},[490,146],[29,12513,1144],{"className":12514},[109,146],[29,12516,154],{"className":12517},[153],[29,12519,12521],{"className":12520},[126],[29,12522,12524],{"className":12523,"style":1760},[130],[29,12525],{},[29,12527],{"className":12528,"style":367},[171],[29,12530,508],{"className":12531},[371],[29,12533],{"className":12534,"style":367},[171],[29,12536,12538,12541,12544,12584,12644,12686,12689,12692,12732,12792,12832,12861,12864,12867],{"className":12537},[100],[29,12539],{"className":12540,"style":9691},[104],[29,12542,460],{"className":12543},[483],[29,12545,12547,12550],{"className":12546},[109],[29,12548,53],{"className":12549},[109,113],[29,12551,12553],{"className":12552},[117],[29,12554,12556,12576],{"className":12555},[121,122],[29,12557,12559,12573],{"className":12558},[126],[29,12560,12562],{"className":12561,"style":711},[130],[29,12563,12564,12567],{"style":134},[29,12565],{"className":12566,"style":139},[138],[29,12568,12570],{"className":12569},[143,144,145,146],[29,12571,618],{"className":12572},[109,113,146],[29,12574,154],{"className":12575},[153],[29,12577,12579],{"className":12578},[126],[29,12580,12582],{"className":12581,"style":161},[130],[29,12583],{},[29,12585,12587,12590],{"className":12586},[109],[29,12588,2087],{"className":12589,"style":2350},[109,113],[29,12591,12593],{"className":12592},[117],[29,12594,12596,12636],{"className":12595},[121,122],[29,12597,12599,12633],{"className":12598},[126],[29,12600,12602,12613],{"className":12601,"style":2363},[130],[29,12603,12604,12607],{"style":2366},[29,12605],{"className":12606,"style":139},[138],[29,12608,12610],{"className":12609},[143,144,145,146],[29,12611,1028],{"className":12612,"style":1319},[109,113,146],[29,12614,12615,12618],{"style":2378},[29,12616],{"className":12617,"style":139},[138],[29,12619,12621],{"className":12620},[143,144,145,146],[29,12622,12624,12627,12630],{"className":12623},[109,146],[29,12625,460],{"className":12626},[483,146],[29,12628,463],{"className":12629},[109,113,146],[29,12631,466],{"className":12632},[490,146],[29,12634,154],{"className":12635},[153],[29,12637,12639],{"className":12638},[126],[29,12640,12642],{"className":12641,"style":2406},[130],[29,12643],{},[29,12645,12647,12651],{"className":12646},[109],[29,12648,306],{"className":12649,"style":12650},[109,113],"margin-right:0.0077em;",[29,12652,12654],{"className":12653},[117],[29,12655,12657,12678],{"className":12656},[121,122],[29,12658,12660,12675],{"className":12659},[126],[29,12661,12663],{"className":12662,"style":711},[130],[29,12664,12666,12669],{"style":12665},"top:-2.55em;margin-left:-0.0077em;margin-right:0.05em;",[29,12667],{"className":12668,"style":139},[138],[29,12670,12672],{"className":12671},[143,144,145,146],[29,12673,618],{"className":12674},[109,113,146],[29,12676,154],{"className":12677},[153],[29,12679,12681],{"className":12680},[126],[29,12682,12684],{"className":12683,"style":161},[130],[29,12685],{},[29,12687,466],{"className":12688},[490],[29,12690,460],{"className":12691},[483],[29,12693,12695,12698],{"className":12694},[109],[29,12696,6467],{"className":12697},[109,113],[29,12699,12701],{"className":12700},[117],[29,12702,12704,12724],{"className":12703},[121,122],[29,12705,12707,12721],{"className":12706},[126],[29,12708,12710],{"className":12709,"style":341},[130],[29,12711,12712,12715],{"style":134},[29,12713],{"className":12714,"style":139},[138],[29,12716,12718],{"className":12717},[143,144,145,146],[29,12719,296],{"className":12720},[109,113,146],[29,12722,154],{"className":12723},[153],[29,12725,12727],{"className":12726},[126],[29,12728,12730],{"className":12729,"style":161},[130],[29,12731],{},[29,12733,12735,12738],{"className":12734},[109],[29,12736,2087],{"className":12737,"style":2350},[109,113],[29,12739,12741],{"className":12740},[117],[29,12742,12744,12784],{"className":12743},[121,122],[29,12745,12747,12781],{"className":12746},[126],[29,12748,12750,12761],{"className":12749,"style":771},[130],[29,12751,12752,12755],{"style":2646},[29,12753],{"className":12754,"style":139},[138],[29,12756,12758],{"className":12757},[143,144,145,146],[29,12759,1045],{"className":12760,"style":1387},[109,113,146],[29,12762,12763,12766],{"style":786},[29,12764],{"className":12765,"style":139},[138],[29,12767,12769],{"className":12768},[143,144,145,146],[29,12770,12772,12775,12778],{"className":12771},[109,146],[29,12773,460],{"className":12774},[483,146],[29,12776,463],{"className":12777},[109,113,146],[29,12779,466],{"className":12780},[490,146],[29,12782,154],{"className":12783},[153],[29,12785,12787],{"className":12786},[126],[29,12788,12790],{"className":12789,"style":2685},[130],[29,12791],{},[29,12793,12795,12798],{"className":12794},[109],[29,12796,306],{"className":12797,"style":12650},[109,113],[29,12799,12801],{"className":12800},[117],[29,12802,12804,12824],{"className":12803},[121,122],[29,12805,12807,12821],{"className":12806},[126],[29,12808,12810],{"className":12809,"style":341},[130],[29,12811,12812,12815],{"style":12665},[29,12813],{"className":12814,"style":139},[138],[29,12816,12818],{"className":12817},[143,144,145,146],[29,12819,296],{"className":12820},[109,113,146],[29,12822,154],{"className":12823},[153],[29,12825,12827],{"className":12826},[126],[29,12828,12830],{"className":12829,"style":161},[130],[29,12831],{},[29,12833,12835,12838],{"className":12834},[490],[29,12836,466],{"className":12837},[490],[29,12839,12841],{"className":12840},[117],[29,12842,12844],{"className":12843},[121],[29,12845,12847],{"className":12846},[126],[29,12848,12850],{"className":12849,"style":2442},[130],[29,12851,12852,12855],{"style":2378},[29,12853],{"className":12854,"style":139},[138],[29,12856,12858],{"className":12857},[143,144,145,146],[29,12859,1144],{"className":12860},[109,146],[29,12862],{"className":12863,"style":367},[171],[29,12865,508],{"className":12866},[371],[29,12868],{"className":12869,"style":367},[171],[29,12871,12873,12876,12916,12919,12979,13029,13092,13095],{"className":12872},[100],[29,12874],{"className":12875,"style":9691},[104],[29,12877,12879,12882],{"className":12878},[109],[29,12880,53],{"className":12881},[109,113],[29,12883,12885],{"className":12884},[117],[29,12886,12888,12908],{"className":12887},[121,122],[29,12889,12891,12905],{"className":12890},[126],[29,12892,12894],{"className":12893,"style":711},[130],[29,12895,12896,12899],{"style":134},[29,12897],{"className":12898,"style":139},[138],[29,12900,12902],{"className":12901},[143,144,145,146],[29,12903,618],{"className":12904},[109,113,146],[29,12906,154],{"className":12907},[153],[29,12909,12911],{"className":12910},[126],[29,12912,12914],{"className":12913,"style":161},[130],[29,12915],{},[29,12917,460],{"className":12918},[483],[29,12920,12922,12925],{"className":12921},[109],[29,12923,2087],{"className":12924,"style":2350},[109,113],[29,12926,12928],{"className":12927},[117],[29,12929,12931,12971],{"className":12930},[121,122],[29,12932,12934,12968],{"className":12933},[126],[29,12935,12937,12948],{"className":12936,"style":2363},[130],[29,12938,12939,12942],{"style":2366},[29,12940],{"className":12941,"style":139},[138],[29,12943,12945],{"className":12944},[143,144,145,146],[29,12946,1028],{"className":12947,"style":1319},[109,113,146],[29,12949,12950,12953],{"style":2378},[29,12951],{"className":12952,"style":139},[138],[29,12954,12956],{"className":12955},[143,144,145,146],[29,12957,12959,12962,12965],{"className":12958},[109,146],[29,12960,460],{"className":12961},[483,146],[29,12963,463],{"className":12964},[109,113,146],[29,12966,466],{"className":12967},[490,146],[29,12969,154],{"className":12970},[153],[29,12972,12974],{"className":12973},[126],[29,12975,12977],{"className":12976,"style":2406},[130],[29,12978],{},[29,12980,12982,12985],{"className":12981},[109],[29,12983,306],{"className":12984,"style":12650},[109,113],[29,12986,12988],{"className":12987},[117],[29,12989,12991,13020],{"className":12990},[121,122],[29,12992,12994,13017],{"className":12993},[126],[29,12995,12997],{"className":12996,"style":341},[130],[29,12998,12999,13002],{"style":12665},[29,13000],{"className":13001,"style":139},[138],[29,13003,13005],{"className":13004},[143,144,145,146],[29,13006,13008,13011,13014],{"className":13007},[109,146],[29,13009,618],{"className":13010},[109,113,146],[29,13012,12362],{"className":13013},[3262,146],[29,13015,296],{"className":13016},[109,113,146],[29,13018,154],{"className":13019},[153],[29,13021,13023],{"className":13022},[126],[29,13024,13027],{"className":13025,"style":13026},[130],"height:0.2083em;",[29,13028],{},[29,13030,13032,13035],{"className":13031},[109],[29,13033,2087],{"className":13034,"style":2350},[109,113],[29,13036,13038],{"className":13037},[117],[29,13039,13041,13084],{"className":13040},[121,122],[29,13042,13044,13081],{"className":13043},[126],[29,13045,13047,13058],{"className":13046,"style":771},[130],[29,13048,13049,13052],{"style":2646},[29,13050],{"className":13051,"style":139},[138],[29,13053,13055],{"className":13054},[143,144,145,146],[29,13056,1045],{"className":13057,"style":1387},[109,113,146],[29,13059,13060,13063],{"style":786},[29,13061],{"className":13062,"style":139},[138],[29,13064,13066],{"className":13065},[143,144,145,146],[29,13067,13069,13072,13075,13078],{"className":13068},[109,146],[29,13070,460],{"className":13071},[483,146],[29,13073,463],{"className":13074},[109,113,146],[29,13076,466],{"className":13077},[490,146],[29,13079,1144],{"className":13080},[109,146],[29,13082,154],{"className":13083},[153],[29,13085,13087],{"className":13086},[126],[29,13088,13090],{"className":13089,"style":2685},[130],[29,13091],{},[29,13093,466],{"className":13094},[490],[29,13096,13098,13101],{"className":13097},[109],[29,13099,6467],{"className":13100},[109,113],[29,13102,13104],{"className":13103},[117],[29,13105,13107,13138],{"className":13106},[121,122],[29,13108,13110,13135],{"className":13109},[126],[29,13111,13113,13124],{"className":13112,"style":2442},[130],[29,13114,13115,13118],{"style":10134},[29,13116],{"className":13117,"style":139},[138],[29,13119,13121],{"className":13120},[143,144,145,146],[29,13122,296],{"className":13123},[109,113,146],[29,13125,13126,13129],{"style":2378},[29,13127],{"className":13128,"style":139},[138],[29,13130,13132],{"className":13131},[143,144,145,146],[29,13133,1144],{"className":13134},[109,146],[29,13136,154],{"className":13137},[153],[29,13139,13141],{"className":13140},[126],[29,13142,13144],{"className":13143,"style":2955},[130],[29,13145],{},[10,13147,13148,13386,13387,13439,13440,13511],{},[29,13149,13151,13205],{"className":13150},[32],[29,13152,13154],{"className":13153},[36],[38,13155,13156],{"xmlns":40},[42,13157,13158,13202],{},[45,13159,13160,13174,13186],{},[625,13161,13162,13164,13166],{},[51,13163,2087],{},[51,13165,1028],{},[45,13167,13168,13170,13172],{},[59,13169,460],{"stretchy":459},[51,13171,463],{},[59,13173,466],{"stretchy":459},[48,13175,13176,13178],{},[51,13177,306],{},[45,13179,13180,13182,13184],{},[51,13181,618],{},[59,13183,12362],{},[51,13185,296],{},[625,13187,13188,13190,13192],{},[51,13189,2087],{},[51,13191,1045],{},[45,13193,13194,13196,13198,13200],{},[59,13195,460],{"stretchy":459},[51,13197,463],{},[59,13199,466],{"stretchy":459},[51,13201,1144],{"mathvariant":1002},[89,13203,13204],{"encoding":91},"W_q^{(s)} R_{t-i} W_k^{(s)\\top}",[29,13206,13208],{"className":13207,"ariaHidden":61},[96],[29,13209,13211,13214,13274,13323],{"className":13210},[100],[29,13212],{"className":13213,"style":2585},[104],[29,13215,13217,13220],{"className":13216},[109],[29,13218,2087],{"className":13219,"style":2350},[109,113],[29,13221,13223],{"className":13222},[117],[29,13224,13226,13266],{"className":13225},[121,122],[29,13227,13229,13263],{"className":13228},[126],[29,13230,13232,13243],{"className":13231,"style":771},[130],[29,13233,13234,13237],{"style":10243},[29,13235],{"className":13236,"style":139},[138],[29,13238,13240],{"className":13239},[143,144,145,146],[29,13241,1028],{"className":13242,"style":1319},[109,113,146],[29,13244,13245,13248],{"style":786},[29,13246],{"className":13247,"style":139},[138],[29,13249,13251],{"className":13250},[143,144,145,146],[29,13252,13254,13257,13260],{"className":13253},[109,146],[29,13255,460],{"className":13256},[483,146],[29,13258,463],{"className":13259},[109,113,146],[29,13261,466],{"className":13262},[490,146],[29,13264,154],{"className":13265},[153],[29,13267,13269],{"className":13268},[126],[29,13270,13272],{"className":13271,"style":10282},[130],[29,13273],{},[29,13275,13277,13280],{"className":13276},[109],[29,13278,306],{"className":13279,"style":12650},[109,113],[29,13281,13283],{"className":13282},[117],[29,13284,13286,13315],{"className":13285},[121,122],[29,13287,13289,13312],{"className":13288},[126],[29,13290,13292],{"className":13291,"style":341},[130],[29,13293,13294,13297],{"style":12665},[29,13295],{"className":13296,"style":139},[138],[29,13298,13300],{"className":13299},[143,144,145,146],[29,13301,13303,13306,13309],{"className":13302},[109,146],[29,13304,618],{"className":13305},[109,113,146],[29,13307,12362],{"className":13308},[3262,146],[29,13310,296],{"className":13311},[109,113,146],[29,13313,154],{"className":13314},[153],[29,13316,13318],{"className":13317},[126],[29,13319,13321],{"className":13320,"style":13026},[130],[29,13322],{},[29,13324,13326,13329],{"className":13325},[109],[29,13327,2087],{"className":13328,"style":2350},[109,113],[29,13330,13332],{"className":13331},[117],[29,13333,13335,13378],{"className":13334},[121,122],[29,13336,13338,13375],{"className":13337},[126],[29,13339,13341,13352],{"className":13340,"style":771},[130],[29,13342,13343,13346],{"style":2646},[29,13344],{"className":13345,"style":139},[138],[29,13347,13349],{"className":13348},[143,144,145,146],[29,13350,1045],{"className":13351,"style":1387},[109,113,146],[29,13353,13354,13357],{"style":786},[29,13355],{"className":13356,"style":139},[138],[29,13358,13360],{"className":13359},[143,144,145,146],[29,13361,13363,13366,13369,13372],{"className":13362},[109,146],[29,13364,460],{"className":13365},[483,146],[29,13367,463],{"className":13368},[109,113,146],[29,13370,466],{"className":13371},[490,146],[29,13373,1144],{"className":13374},[109,146],[29,13376,154],{"className":13377},[153],[29,13379,13381],{"className":13380},[126],[29,13382,13384],{"className":13383,"style":2685},[130],[29,13385],{}," 跟位置差 ",[29,13388,13390,13408],{"className":13389},[32],[29,13391,13393],{"className":13392},[36],[38,13394,13395],{"xmlns":40},[42,13396,13397,13405],{},[45,13398,13399,13401,13403],{},[51,13400,618],{},[59,13402,12362],{},[51,13404,296],{},[89,13406,13407],{"encoding":91},"t - i",[29,13409,13411,13429],{"className":13410,"ariaHidden":61},[96],[29,13412,13414,13417,13420,13423,13426],{"className":13413},[100],[29,13415],{"className":13416,"style":3251},[104],[29,13418,618],{"className":13419},[109,113],[29,13421],{"className":13422,"style":3258},[171],[29,13424,12362],{"className":13425},[3262],[29,13427],{"className":13428,"style":3258},[171],[29,13430,13432,13436],{"className":13431},[100],[29,13433],{"className":13434,"style":13435},[104],"height:0.6595em;",[29,13437,296],{"className":13438},[109,113]," 相关，无法合并成一个固定矩阵。放弃 RoPE 换 ALIBI 之类的 attention bias 是一种办法，但实验显示明显不如 RoPE。最后 MLA 采取的是混合方案：每个 head 的 Q、K 新增 ",[29,13441,13443,13462],{"className":13442},[32],[29,13444,13446],{"className":13445},[36],[38,13447,13448],{"xmlns":40},[42,13449,13450,13459],{},[45,13451,13452],{},[48,13453,13454,13456],{},[51,13455,309],{},[51,13457,13458],{},"r",[89,13460,13461],{"encoding":91},"d_r",[29,13463,13465],{"className":13464,"ariaHidden":61},[96],[29,13466,13468,13471],{"className":13467},[100],[29,13469],{"className":13470,"style":3098},[104],[29,13472,13474,13477],{"className":13473},[109],[29,13475,309],{"className":13476},[109,113],[29,13478,13480],{"className":13479},[117],[29,13481,13483,13503],{"className":13482},[121,122],[29,13484,13486,13500],{"className":13485},[126],[29,13487,13489],{"className":13488,"style":3172},[130],[29,13490,13491,13494],{"style":134},[29,13492],{"className":13493,"style":139},[138],[29,13495,13497],{"className":13496},[143,144,145,146],[29,13498,13458],{"className":13499,"style":4465},[109,113,146],[29,13501,154],{"className":13502},[153],[29,13504,13506],{"className":13505},[126],[29,13507,13509],{"className":13508,"style":161},[130],[29,13510],{}," 个维度专门用来加 RoPE，其中 K 新增的这部分在 head 间共享。",[10,13513,13514,13515,13584,13585,13686,13687,13780],{},"这样没有 RoPE 的维度继续走前面的压缩，带 RoPE 的维度补充位置信息。",[29,13516,13518,13535],{"className":13517},[32],[29,13519,13521],{"className":13520},[36],[38,13522,13523],{"xmlns":40},[42,13524,13525,13533],{},[45,13526,13527],{},[48,13528,13529,13531],{},[51,13530,309],{},[51,13532,13458],{},[89,13534,13461],{"encoding":91},[29,13536,13538],{"className":13537,"ariaHidden":61},[96],[29,13539,13541,13544],{"className":13540},[100],[29,13542],{"className":13543,"style":3098},[104],[29,13545,13547,13550],{"className":13546},[109],[29,13548,309],{"className":13549},[109,113],[29,13551,13553],{"className":13552},[117],[29,13554,13556,13576],{"className":13555},[121,122],[29,13557,13559,13573],{"className":13558},[126],[29,13560,13562],{"className":13561,"style":3172},[130],[29,13563,13564,13567],{"style":134},[29,13565],{"className":13566,"style":139},[138],[29,13568,13570],{"className":13569},[143,144,145,146],[29,13571,13458],{"className":13572,"style":4465},[109,113,146],[29,13574,154],{"className":13575},[153],[29,13577,13579],{"className":13578},[126],[29,13580,13582],{"className":13581,"style":161},[130],[29,13583],{}," 取 ",[29,13586,13588,13615],{"className":13587},[32],[29,13589,13591],{"className":13590},[36],[38,13592,13593],{"xmlns":40},[42,13594,13595,13612],{},[45,13596,13597,13603,13605,13607,13609],{},[48,13598,13599,13601],{},[51,13600,309],{},[51,13602,1045],{},[51,13604,3083],{"mathvariant":1002},[55,13606,69],{},[59,13608,508],{},[55,13610,13611],{},"64",[89,13613,13614],{"encoding":91},"d_k\u002F2 = 64",[29,13616,13618,13677],{"className":13617,"ariaHidden":61},[96],[29,13619,13621,13624,13664,13668,13671,13674],{"className":13620},[100],[29,13622],{"className":13623,"style":479},[104],[29,13625,13627,13630],{"className":13626},[109],[29,13628,309],{"className":13629},[109,113],[29,13631,13633],{"className":13632},[117],[29,13634,13636,13656],{"className":13635},[121,122],[29,13637,13639,13653],{"className":13638},[126],[29,13640,13642],{"className":13641,"style":253},[130],[29,13643,13644,13647],{"style":134},[29,13645],{"className":13646,"style":139},[138],[29,13648,13650],{"className":13649},[143,144,145,146],[29,13651,1045],{"className":13652,"style":1387},[109,113,146],[29,13654,154],{"className":13655},[153],[29,13657,13659],{"className":13658},[126],[29,13660,13662],{"className":13661,"style":161},[130],[29,13663],{},[29,13665,13667],{"className":13666},[109],"\u002F2",[29,13669],{"className":13670,"style":367},[171],[29,13672,508],{"className":13673},[371],[29,13675],{"className":13676,"style":367},[171],[29,13678,13680,13683],{"className":13679},[100],[29,13681],{"className":13682,"style":3272},[104],[29,13684,13611],{"className":13685},[109],"，相比 ",[29,13688,13690,13713],{"className":13689},[32],[29,13691,13693],{"className":13692},[36],[38,13694,13695],{"xmlns":40},[42,13696,13697,13710],{},[45,13698,13699,13705,13707],{},[48,13700,13701,13703],{},[51,13702,309],{},[51,13704,6467],{},[59,13706,508],{},[55,13708,13709],{},"512",[89,13711,13712],{"encoding":91},"d_c = 512",[29,13714,13716,13771],{"className":13715,"ariaHidden":61},[96],[29,13717,13719,13722,13762,13765,13768],{"className":13718},[100],[29,13720],{"className":13721,"style":3098},[104],[29,13723,13725,13728],{"className":13724},[109],[29,13726,309],{"className":13727},[109,113],[29,13729,13731],{"className":13730},[117],[29,13732,13734,13754],{"className":13733},[121,122],[29,13735,13737,13751],{"className":13736},[126],[29,13738,13740],{"className":13739,"style":3172},[130],[29,13741,13742,13745],{"style":134},[29,13743],{"className":13744,"style":139},[138],[29,13746,13748],{"className":13747},[143,144,145,146],[29,13749,6467],{"className":13750},[109,113,146],[29,13752,154],{"className":13753},[153],[29,13755,13757],{"className":13756},[126],[29,13758,13760],{"className":13759,"style":161},[130],[29,13761],{},[29,13763],{"className":13764,"style":367},[171],[29,13766,508],{"className":13767},[371],[29,13769],{"className":13770,"style":367},[171],[29,13772,13774,13777],{"className":13773},[100],[29,13775],{"className":13776,"style":3272},[104],[29,13778,13709],{"className":13779},[109],"，增加幅度不大。",[10,13782,13783,13784,13874,13875,13966,13967,14100,14101,14171],{},"还有一个细节：MLA 最终版把 Q 的输入也改成了低秩投影。这跟减少 KV Cache 无关，主要是为了减少训练期间的参数量和梯度所占显存。",[29,13785,13787,13809],{"className":13786},[32],[29,13788,13790],{"className":13789},[36],[38,13791,13792],{"xmlns":40},[42,13793,13794,13806],{},[45,13795,13796],{},[625,13797,13798,13800,13802],{},[51,13799,309],{},[51,13801,6467],{},[59,13803,13805],{"mathvariant":1002,"lspace":13804,"rspace":13804},"0em","′",[89,13807,13808],{"encoding":91},"d_c'",[29,13810,13812],{"className":13811,"ariaHidden":61},[96],[29,13813,13815,13819],{"className":13814},[100],[29,13816],{"className":13817,"style":13818},[104],"height:0.9989em;vertical-align:-0.247em;",[29,13820,13822,13825],{"className":13821},[109],[29,13823,309],{"className":13824},[109,113],[29,13826,13828],{"className":13827},[117],[29,13829,13831,13866],{"className":13830},[121,122],[29,13832,13834,13863],{"className":13833},[126],[29,13835,13838,13849],{"className":13836,"style":13837},[130],"height:0.7519em;",[29,13839,13840,13843],{"style":10134},[29,13841],{"className":13842,"style":139},[138],[29,13844,13846],{"className":13845},[143,144,145,146],[29,13847,6467],{"className":13848},[109,113,146],[29,13850,13851,13854],{"style":403},[29,13852],{"className":13853,"style":139},[138],[29,13855,13857],{"className":13856},[143,144,145,146],[29,13858,13860],{"className":13859},[109,146],[29,13861,13805],{"className":13862},[109,146],[29,13864,154],{"className":13865},[153],[29,13867,13869],{"className":13868},[126],[29,13870,13872],{"className":13871,"style":2955},[130],[29,13873],{}," 取 1536，和 ",[29,13876,13878,13899],{"className":13877},[32],[29,13879,13881],{"className":13880},[36],[38,13882,13883],{"xmlns":40},[42,13884,13885,13897],{},[45,13886,13887,13893,13895],{},[48,13888,13889,13891],{},[51,13890,309],{},[51,13892,6467],{},[59,13894,508],{},[55,13896,13709],{},[89,13898,13712],{"encoding":91},[29,13900,13902,13957],{"className":13901,"ariaHidden":61},[96],[29,13903,13905,13908,13948,13951,13954],{"className":13904},[100],[29,13906],{"className":13907,"style":3098},[104],[29,13909,13911,13914],{"className":13910},[109],[29,13912,309],{"className":13913},[109,113],[29,13915,13917],{"className":13916},[117],[29,13918,13920,13940],{"className":13919},[121,122],[29,13921,13923,13937],{"className":13922},[126],[29,13924,13926],{"className":13925,"style":3172},[130],[29,13927,13928,13931],{"style":134},[29,13929],{"className":13930,"style":139},[138],[29,13932,13934],{"className":13933},[143,144,145,146],[29,13935,6467],{"className":13936},[109,113,146],[29,13938,154],{"className":13939},[153],[29,13941,13943],{"className":13942},[126],[29,13944,13946],{"className":13945,"style":161},[130],[29,13947],{},[29,13949],{"className":13950,"style":367},[171],[29,13952,508],{"className":13953},[371],[29,13955],{"className":13956,"style":367},[171],[29,13958,13960,13963],{"className":13959},[100],[29,13961],{"className":13962,"style":3272},[104],[29,13964,13709],{"className":13965},[109]," 不同。于是解码阶段的 MLA 是 MQA 形式，Q、K 的 head size 变成 ",[29,13968,13970,13996],{"className":13969},[32],[29,13971,13973],{"className":13972},[36],[38,13974,13975],{"xmlns":40},[42,13976,13977,13993],{},[45,13978,13979,13985,13987],{},[48,13980,13981,13983],{},[51,13982,309],{},[51,13984,6467],{},[59,13986,3236],{},[48,13988,13989,13991],{},[51,13990,309],{},[51,13992,13458],{},[89,13994,13995],{"encoding":91},"d_c + d_r",[29,13997,13999,14054],{"className":13998,"ariaHidden":61},[96],[29,14000,14002,14005,14045,14048,14051],{"className":14001},[100],[29,14003],{"className":14004,"style":3098},[104],[29,14006,14008,14011],{"className":14007},[109],[29,14009,309],{"className":14010},[109,113],[29,14012,14014],{"className":14013},[117],[29,14015,14017,14037],{"className":14016},[121,122],[29,14018,14020,14034],{"className":14019},[126],[29,14021,14023],{"className":14022,"style":3172},[130],[29,14024,14025,14028],{"style":134},[29,14026],{"className":14027,"style":139},[138],[29,14029,14031],{"className":14030},[143,144,145,146],[29,14032,6467],{"className":14033},[109,113,146],[29,14035,154],{"className":14036},[153],[29,14038,14040],{"className":14039},[126],[29,14041,14043],{"className":14042,"style":161},[130],[29,14044],{},[29,14046],{"className":14047,"style":3258},[171],[29,14049,3236],{"className":14050},[3262],[29,14052],{"className":14053,"style":3258},[171],[29,14055,14057,14060],{"className":14056},[100],[29,14058],{"className":14059,"style":3098},[104],[29,14061,14063,14066],{"className":14062},[109],[29,14064,309],{"className":14065},[109,113],[29,14067,14069],{"className":14068},[117],[29,14070,14072,14092],{"className":14071},[121,122],[29,14073,14075,14089],{"className":14074},[126],[29,14076,14078],{"className":14077,"style":3172},[130],[29,14079,14080,14083],{"style":134},[29,14081],{"className":14082,"style":139},[138],[29,14084,14086],{"className":14085},[143,144,145,146],[29,14087,13458],{"className":14088,"style":4465},[109,113,146],[29,14090,154],{"className":14091},[153],[29,14093,14095],{"className":14094},[126],[29,14096,14098],{"className":14097,"style":161},[130],[29,14099],{},"，是 ",[29,14102,14104,14122],{"className":14103},[32],[29,14105,14107],{"className":14106},[36],[38,14108,14109],{"xmlns":40},[42,14110,14111,14119],{},[45,14112,14113],{},[48,14114,14115,14117],{},[51,14116,309],{},[51,14118,1045],{},[89,14120,14121],{"encoding":91},"d_k",[29,14123,14125],{"className":14124,"ariaHidden":61},[96],[29,14126,14128,14131],{"className":14127},[100],[29,14129],{"className":14130,"style":3098},[104],[29,14132,14134,14137],{"className":14133},[109],[29,14135,309],{"className":14136},[109,113],[29,14138,14140],{"className":14139},[117],[29,14141,14143,14163],{"className":14142},[121,122],[29,14144,14146,14160],{"className":14145},[126],[29,14147,14149],{"className":14148,"style":253},[130],[29,14150,14151,14154],{"style":134},[29,14152],{"className":14153,"style":139},[138],[29,14155,14157],{"className":14156},[143,144,145,146],[29,14158,1045],{"className":14159,"style":1387},[109,113,146],[29,14161,154],{"className":14162},[153],[29,14164,14166],{"className":14165},[126],[29,14167,14169],{"className":14168,"style":161},[130],[29,14170],{}," 的 4 倍。",[10,14173,14174],{},[14,14175,14176],{},"解码的计算量其实是增加的。",[10,14178,14179,14180,14232,14233,14284,14285,14377,14378,6069,14469,14561,14562,14614,14615,14643],{},"用 DeepSeek-V2 的配置把这笔账算一遍。",[29,14181,14183,14202],{"className":14182},[32],[29,14184,14186],{"className":14185},[36],[38,14187,14188],{"xmlns":40},[42,14189,14190,14199],{},[45,14191,14192,14194,14196],{},[51,14193,3517],{},[59,14195,508],{},[55,14197,14198],{},"60",[89,14200,14201],{"encoding":91},"L = 60",[29,14203,14205,14223],{"className":14204,"ariaHidden":61},[96],[29,14206,14208,14211,14214,14217,14220],{"className":14207},[100],[29,14209],{"className":14210,"style":3718},[104],[29,14212,3517],{"className":14213},[109,113],[29,14215],{"className":14216,"style":367},[171],[29,14218,508],{"className":14219},[371],[29,14221],{"className":14222,"style":367},[171],[29,14224,14226,14229],{"className":14225},[100],[29,14227],{"className":14228,"style":3272},[104],[29,14230,14198],{"className":14231},[109]," 层，",[29,14234,14236,14254],{"className":14235},[32],[29,14237,14239],{"className":14238},[36],[38,14240,14241],{"xmlns":40},[42,14242,14243,14251],{},[45,14244,14245,14247,14249],{},[51,14246,428],{},[59,14248,508],{},[55,14250,3840],{},[89,14252,14253],{"encoding":91},"h = 128",[29,14255,14257,14275],{"className":14256,"ariaHidden":61},[96],[29,14258,14260,14263,14266,14269,14272],{"className":14259},[100],[29,14261],{"className":14262,"style":440},[104],[29,14264,428],{"className":14265},[109,113],[29,14267],{"className":14268,"style":367},[171],[29,14270,508],{"className":14271},[371],[29,14273],{"className":14274,"style":367},[171],[29,14276,14278,14281],{"className":14277},[100],[29,14279],{"className":14280,"style":3272},[104],[29,14282,3840],{"className":14283},[109]," 个 head，",[29,14286,14288,14310],{"className":14287},[32],[29,14289,14291],{"className":14290},[36],[38,14292,14293],{"xmlns":40},[42,14294,14295,14307],{},[45,14296,14297,14303,14305],{},[48,14298,14299,14301],{},[51,14300,309],{},[51,14302,1045],{},[59,14304,508],{},[55,14306,3840],{},[89,14308,14309],{"encoding":91},"d_k = 128",[29,14311,14313,14368],{"className":14312,"ariaHidden":61},[96],[29,14314,14316,14319,14359,14362,14365],{"className":14315},[100],[29,14317],{"className":14318,"style":3098},[104],[29,14320,14322,14325],{"className":14321},[109],[29,14323,309],{"className":14324},[109,113],[29,14326,14328],{"className":14327},[117],[29,14329,14331,14351],{"className":14330},[121,122],[29,14332,14334,14348],{"className":14333},[126],[29,14335,14337],{"className":14336,"style":253},[130],[29,14338,14339,14342],{"style":134},[29,14340],{"className":14341,"style":139},[138],[29,14343,14345],{"className":14344},[143,144,145,146],[29,14346,1045],{"className":14347,"style":1387},[109,113,146],[29,14349,154],{"className":14350},[153],[29,14352,14354],{"className":14353},[126],[29,14355,14357],{"className":14356,"style":161},[130],[29,14358],{},[29,14360],{"className":14361,"style":367},[171],[29,14363,508],{"className":14364},[371],[29,14366],{"className":14367,"style":367},[171],[29,14369,14371,14374],{"className":14370},[100],[29,14372],{"className":14373,"style":3272},[104],[29,14375,3840],{"className":14376},[109],"，MLA 这边 ",[29,14379,14381,14402],{"className":14380},[32],[29,14382,14384],{"className":14383},[36],[38,14385,14386],{"xmlns":40},[42,14387,14388,14400],{},[45,14389,14390,14396,14398],{},[48,14391,14392,14394],{},[51,14393,309],{},[51,14395,6467],{},[59,14397,508],{},[55,14399,13709],{},[89,14401,13712],{"encoding":91},[29,14403,14405,14460],{"className":14404,"ariaHidden":61},[96],[29,14406,14408,14411,14451,14454,14457],{"className":14407},[100],[29,14409],{"className":14410,"style":3098},[104],[29,14412,14414,14417],{"className":14413},[109],[29,14415,309],{"className":14416},[109,113],[29,14418,14420],{"className":14419},[117],[29,14421,14423,14443],{"className":14422},[121,122],[29,14424,14426,14440],{"className":14425},[126],[29,14427,14429],{"className":14428,"style":3172},[130],[29,14430,14431,14434],{"style":134},[29,14432],{"className":14433,"style":139},[138],[29,14435,14437],{"className":14436},[143,144,145,146],[29,14438,6467],{"className":14439},[109,113,146],[29,14441,154],{"className":14442},[153],[29,14444,14446],{"className":14445},[126],[29,14447,14449],{"className":14448,"style":161},[130],[29,14450],{},[29,14452],{"className":14453,"style":367},[171],[29,14455,508],{"className":14456},[371],[29,14458],{"className":14459,"style":367},[171],[29,14461,14463,14466],{"className":14462},[100],[29,14464],{"className":14465,"style":3272},[104],[29,14467,13709],{"className":14468},[109],[29,14470,14472,14494],{"className":14471},[32],[29,14473,14475],{"className":14474},[36],[38,14476,14477],{"xmlns":40},[42,14478,14479,14491],{},[45,14480,14481,14487,14489],{},[48,14482,14483,14485],{},[51,14484,309],{},[51,14486,13458],{},[59,14488,508],{},[55,14490,13611],{},[89,14492,14493],{"encoding":91},"d_r = 64",[29,14495,14497,14552],{"className":14496,"ariaHidden":61},[96],[29,14498,14500,14503,14543,14546,14549],{"className":14499},[100],[29,14501],{"className":14502,"style":3098},[104],[29,14504,14506,14509],{"className":14505},[109],[29,14507,309],{"className":14508},[109,113],[29,14510,14512],{"className":14511},[117],[29,14513,14515,14535],{"className":14514},[121,122],[29,14516,14518,14532],{"className":14517},[126],[29,14519,14521],{"className":14520,"style":3172},[130],[29,14522,14523,14526],{"style":134},[29,14524],{"className":14525,"style":139},[138],[29,14527,14529],{"className":14528},[143,144,145,146],[29,14530,13458],{"className":14531,"style":4465},[109,113,146],[29,14533,154],{"className":14534},[153],[29,14536,14538],{"className":14537},[126],[29,14539,14541],{"className":14540,"style":161},[130],[29,14542],{},[29,14544],{"className":14545,"style":367},[171],[29,14547,508],{"className":14548},[371],[29,14550],{"className":14551,"style":367},[171],[29,14553,14555,14558],{"className":14554},[100],[29,14556],{"className":14557,"style":3272},[104],[29,14559,13611],{"className":14560},[109],"，BF16 存储，所以每参数 ",[29,14563,14565,14584],{"className":14564},[32],[29,14566,14568],{"className":14567},[36],[38,14569,14570],{"xmlns":40},[42,14571,14572,14581],{},[45,14573,14574,14577,14579],{},[51,14575,14576],{},"b",[59,14578,508],{},[55,14580,69],{},[89,14582,14583],{"encoding":91},"b = 2",[29,14585,14587,14605],{"className":14586,"ariaHidden":61},[96],[29,14588,14590,14593,14596,14599,14602],{"className":14589},[100],[29,14591],{"className":14592,"style":440},[104],[29,14594,14576],{"className":14595},[109,113],[29,14597],{"className":14598,"style":367},[171],[29,14600,508],{"className":14601},[371],[29,14603],{"className":14604,"style":367},[171],[29,14606,14608,14611],{"className":14607},[100],[29,14609],{"className":14610,"style":3272},[104],[29,14612,69],{"className":14613},[109]," 字节。decode 到第 ",[29,14616,14618,14631],{"className":14617},[32],[29,14619,14621],{"className":14620},[36],[38,14622,14623],{"xmlns":40},[42,14624,14625,14629],{},[45,14626,14627],{},[51,14628,3534],{},[89,14630,3534],{"encoding":91},[29,14632,14634],{"className":14633,"ariaHidden":61},[96],[29,14635,14637,14640],{"className":14636},[100],[29,14638],{"className":14639,"style":3718},[104],[29,14641,3534],{"className":14642,"style":3667},[109,113]," 个位置时，每 token 每层：",[14645,14646,14647,14662],"table",{},[14648,14649,14650],"thead",{},[14651,14652,14653,14656,14659],"tr",{},[14654,14655],"th",{},[14654,14657,14658],{},"KV 读取",[14654,14660,14661],{},"attention FLOPs",[14663,14664,14665,14851],"tbody",{},[14651,14666,14667,14671,14763],{},[14668,14669,14670],"td",{},"MHA",[14668,14672,14673],{},[29,14674,14676,14702],{"className":14675},[32],[29,14677,14679],{"className":14678},[36],[38,14680,14681],{"xmlns":40},[42,14682,14683,14699],{},[45,14684,14685,14687,14689,14695,14697],{},[55,14686,69],{},[51,14688,428],{},[48,14690,14691,14693],{},[51,14692,309],{},[51,14694,1045],{},[51,14696,3534],{},[51,14698,14576],{},[89,14700,14701],{"encoding":91},"2 h d_k N b",[29,14703,14705],{"className":14704,"ariaHidden":61},[96],[29,14706,14708,14711,14714,14717,14757,14760],{"className":14707},[100],[29,14709],{"className":14710,"style":3098},[104],[29,14712,69],{"className":14713},[109],[29,14715,428],{"className":14716},[109,113],[29,14718,14720,14723],{"className":14719},[109],[29,14721,309],{"className":14722},[109,113],[29,14724,14726],{"className":14725},[117],[29,14727,14729,14749],{"className":14728},[121,122],[29,14730,14732,14746],{"className":14731},[126],[29,14733,14735],{"className":14734,"style":253},[130],[29,14736,14737,14740],{"style":134},[29,14738],{"className":14739,"style":139},[138],[29,14741,14743],{"className":14742},[143,144,145,146],[29,14744,1045],{"className":14745,"style":1387},[109,113,146],[29,14747,154],{"className":14748},[153],[29,14750,14752],{"className":14751},[126],[29,14753,14755],{"className":14754,"style":161},[130],[29,14756],{},[29,14758,3534],{"className":14759,"style":3667},[109,113],[29,14761,14576],{"className":14762},[109,113],[14668,14764,14765],{},[29,14766,14768,14793],{"className":14767},[32],[29,14769,14771],{"className":14770},[36],[38,14772,14773],{"xmlns":40},[42,14774,14775,14790],{},[45,14776,14777,14780,14782,14788],{},[55,14778,14779],{},"4",[51,14781,428],{},[48,14783,14784,14786],{},[51,14785,309],{},[51,14787,1045],{},[51,14789,3534],{},[89,14791,14792],{"encoding":91},"4 h d_k N",[29,14794,14796],{"className":14795,"ariaHidden":61},[96],[29,14797,14799,14802,14805,14808,14848],{"className":14798},[100],[29,14800],{"className":14801,"style":3098},[104],[29,14803,14779],{"className":14804},[109],[29,14806,428],{"className":14807},[109,113],[29,14809,14811,14814],{"className":14810},[109],[29,14812,309],{"className":14813},[109,113],[29,14815,14817],{"className":14816},[117],[29,14818,14820,14840],{"className":14819},[121,122],[29,14821,14823,14837],{"className":14822},[126],[29,14824,14826],{"className":14825,"style":253},[130],[29,14827,14828,14831],{"style":134},[29,14829],{"className":14830,"style":139},[138],[29,14832,14834],{"className":14833},[143,144,145,146],[29,14835,1045],{"className":14836,"style":1387},[109,113,146],[29,14838,154],{"className":14839},[153],[29,14841,14843],{"className":14842},[126],[29,14844,14846],{"className":14845,"style":161},[130],[29,14847],{},[29,14849,3534],{"className":14850,"style":3667},[109,113],[14651,14852,14853,14856,15011],{},[14668,14854,14855],{},"MLA decode",[14668,14857,14858],{},[29,14859,14861,14895],{"className":14860},[32],[29,14862,14864],{"className":14863},[36],[38,14865,14866],{"xmlns":40},[42,14867,14868,14892],{},[45,14869,14870,14872,14878,14880,14886,14888,14890],{},[59,14871,460],{"stretchy":459},[48,14873,14874,14876],{},[51,14875,309],{},[51,14877,6467],{},[59,14879,3236],{},[48,14881,14882,14884],{},[51,14883,309],{},[51,14885,13458],{},[59,14887,466],{"stretchy":459},[51,14889,3534],{},[51,14891,14576],{},[89,14893,14894],{"encoding":91},"(d_c + d_r) N b",[29,14896,14898,14956],{"className":14897,"ariaHidden":61},[96],[29,14899,14901,14904,14907,14947,14950,14953],{"className":14900},[100],[29,14902],{"className":14903,"style":479},[104],[29,14905,460],{"className":14906},[483],[29,14908,14910,14913],{"className":14909},[109],[29,14911,309],{"className":14912},[109,113],[29,14914,14916],{"className":14915},[117],[29,14917,14919,14939],{"className":14918},[121,122],[29,14920,14922,14936],{"className":14921},[126],[29,14923,14925],{"className":14924,"style":3172},[130],[29,14926,14927,14930],{"style":134},[29,14928],{"className":14929,"style":139},[138],[29,14931,14933],{"className":14932},[143,144,145,146],[29,14934,6467],{"className":14935},[109,113,146],[29,14937,154],{"className":14938},[153],[29,14940,14942],{"className":14941},[126],[29,14943,14945],{"className":14944,"style":161},[130],[29,14946],{},[29,14948],{"className":14949,"style":3258},[171],[29,14951,3236],{"className":14952},[3262],[29,14954],{"className":14955,"style":3258},[171],[29,14957,14959,14962,15002,15005,15008],{"className":14958},[100],[29,14960],{"className":14961,"style":479},[104],[29,14963,14965,14968],{"className":14964},[109],[29,14966,309],{"className":14967},[109,113],[29,14969,14971],{"className":14970},[117],[29,14972,14974,14994],{"className":14973},[121,122],[29,14975,14977,14991],{"className":14976},[126],[29,14978,14980],{"className":14979,"style":3172},[130],[29,14981,14982,14985],{"style":134},[29,14983],{"className":14984,"style":139},[138],[29,14986,14988],{"className":14987},[143,144,145,146],[29,14989,13458],{"className":14990,"style":4465},[109,113,146],[29,14992,154],{"className":14993},[153],[29,14995,14997],{"className":14996},[126],[29,14998,15000],{"className":14999,"style":161},[130],[29,15001],{},[29,15003,466],{"className":15004},[490],[29,15006,3534],{"className":15007,"style":3667},[109,113],[29,15009,14576],{"className":15010},[109,113],[14668,15012,15013],{},[29,15014,15016,15066],{"className":15015},[32],[29,15017,15019],{"className":15018},[36],[38,15020,15021],{"xmlns":40},[42,15022,15023,15063],{},[45,15024,15025,15027,15029,15031,15037,15039,15045,15047,15049,15051,15053,15055,15061],{},[55,15026,69],{},[51,15028,428],{},[59,15030,460],{"stretchy":459},[48,15032,15033,15035],{},[51,15034,309],{},[51,15036,6467],{},[59,15038,3236],{},[48,15040,15041,15043],{},[51,15042,309],{},[51,15044,13458],{},[59,15046,466],{"stretchy":459},[51,15048,3534],{},[59,15050,3236],{},[55,15052,69],{},[51,15054,428],{},[48,15056,15057,15059],{},[51,15058,309],{},[51,15060,6467],{},[51,15062,3534],{},[89,15064,15065],{"encoding":91},"2h(d_c+d_r)N + 2hd_cN",[29,15067,15069,15133,15194],{"className":15068,"ariaHidden":61},[96],[29,15070,15072,15075,15078,15081,15084,15124,15127,15130],{"className":15071},[100],[29,15073],{"className":15074,"style":479},[104],[29,15076,69],{"className":15077},[109],[29,15079,428],{"className":15080},[109,113],[29,15082,460],{"className":15083},[483],[29,15085,15087,15090],{"className":15086},[109],[29,15088,309],{"className":15089},[109,113],[29,15091,15093],{"className":15092},[117],[29,15094,15096,15116],{"className":15095},[121,122],[29,15097,15099,15113],{"className":15098},[126],[29,15100,15102],{"className":15101,"style":3172},[130],[29,15103,15104,15107],{"style":134},[29,15105],{"className":15106,"style":139},[138],[29,15108,15110],{"className":15109},[143,144,145,146],[29,15111,6467],{"className":15112},[109,113,146],[29,15114,154],{"className":15115},[153],[29,15117,15119],{"className":15118},[126],[29,15120,15122],{"className":15121,"style":161},[130],[29,15123],{},[29,15125],{"className":15126,"style":3258},[171],[29,15128,3236],{"className":15129},[3262],[29,15131],{"className":15132,"style":3258},[171],[29,15134,15136,15139,15179,15182,15185,15188,15191],{"className":15135},[100],[29,15137],{"className":15138,"style":479},[104],[29,15140,15142,15145],{"className":15141},[109],[29,15143,309],{"className":15144},[109,113],[29,15146,15148],{"className":15147},[117],[29,15149,15151,15171],{"className":15150},[121,122],[29,15152,15154,15168],{"className":15153},[126],[29,15155,15157],{"className":15156,"style":3172},[130],[29,15158,15159,15162],{"style":134},[29,15160],{"className":15161,"style":139},[138],[29,15163,15165],{"className":15164},[143,144,145,146],[29,15166,13458],{"className":15167,"style":4465},[109,113,146],[29,15169,154],{"className":15170},[153],[29,15172,15174],{"className":15173},[126],[29,15175,15177],{"className":15176,"style":161},[130],[29,15178],{},[29,15180,466],{"className":15181},[490],[29,15183,3534],{"className":15184,"style":3667},[109,113],[29,15186],{"className":15187,"style":3258},[171],[29,15189,3236],{"className":15190},[3262],[29,15192],{"className":15193,"style":3258},[171],[29,15195,15197,15200,15203,15206,15246],{"className":15196},[100],[29,15198],{"className":15199,"style":3098},[104],[29,15201,69],{"className":15202},[109],[29,15204,428],{"className":15205},[109,113],[29,15207,15209,15212],{"className":15208},[109],[29,15210,309],{"className":15211},[109,113],[29,15213,15215],{"className":15214},[117],[29,15216,15218,15238],{"className":15217},[121,122],[29,15219,15221,15235],{"className":15220},[126],[29,15222,15224],{"className":15223,"style":3172},[130],[29,15225,15226,15229],{"style":134},[29,15227],{"className":15228,"style":139},[138],[29,15230,15232],{"className":15231},[143,144,145,146],[29,15233,6467],{"className":15234},[109,113,146],[29,15236,154],{"className":15237},[153],[29,15239,15241],{"className":15240},[126],[29,15242,15244],{"className":15243,"style":161},[130],[29,15245],{},[29,15247,3534],{"className":15248,"style":3667},[109,113],[10,15250,15251],{},"代进数字：",[14645,15253,15254,15267],{},[14648,15255,15256],{},[14651,15257,15258,15260,15262,15264],{},[14654,15259],{},[14654,15261,14670],{},[14654,15263,6444],{},[14654,15265,15266],{},"比值",[14663,15268,15269,15354],{},[14651,15270,15271,15274,15312,15349],{},[14668,15272,15273],{},"KV 读取 \u002F 层 \u002F token",[14668,15275,15276,15311],{},[29,15277,15279,15296],{"className":15278},[32],[29,15280,15282],{"className":15281},[36],[38,15283,15284],{"xmlns":40},[42,15285,15286,15293],{},[45,15287,15288,15291],{},[55,15289,15290],{},"65536",[51,15292,3534],{},[89,15294,15295],{"encoding":91},"65536N",[29,15297,15299],{"className":15298,"ariaHidden":61},[96],[29,15300,15302,15305,15308],{"className":15301},[100],[29,15303],{"className":15304,"style":3718},[104],[29,15306,15290],{"className":15307},[109],[29,15309,3534],{"className":15310,"style":3667},[109,113],"B",[14668,15313,15314,15311],{},[29,15315,15317,15334],{"className":15316},[32],[29,15318,15320],{"className":15319},[36],[38,15321,15322],{"xmlns":40},[42,15323,15324,15331],{},[45,15325,15326,15329],{},[55,15327,15328],{},"1152",[51,15330,3534],{},[89,15332,15333],{"encoding":91},"1152N",[29,15335,15337],{"className":15336,"ariaHidden":61},[96],[29,15338,15340,15343,15346],{"className":15339},[100],[29,15341],{"className":15342,"style":3718},[104],[29,15344,15328],{"className":15345},[109],[29,15347,3534],{"className":15348,"style":3667},[109,113],[14668,15350,15351],{},[14,15352,15353],{},"1 : 56.9",[14651,15355,15356,15359,15394,15431],{},[14668,15357,15358],{},"attention FLOPs \u002F 层 \u002F token",[14668,15360,15361],{},[29,15362,15364,15379],{"className":15363},[32],[29,15365,15367],{"className":15366},[36],[38,15368,15369],{"xmlns":40},[42,15370,15371,15377],{},[45,15372,15373,15375],{},[55,15374,15290],{},[51,15376,3534],{},[89,15378,15295],{"encoding":91},[29,15380,15382],{"className":15381,"ariaHidden":61},[96],[29,15383,15385,15388,15391],{"className":15384},[100],[29,15386],{"className":15387,"style":3718},[104],[29,15389,15290],{"className":15390},[109],[29,15392,3534],{"className":15393,"style":3667},[109,113],[14668,15395,15396],{},[29,15397,15399,15416],{"className":15398},[32],[29,15400,15402],{"className":15401},[36],[38,15403,15404],{"xmlns":40},[42,15405,15406,15413],{},[45,15407,15408,15411],{},[55,15409,15410],{},"278528",[51,15412,3534],{},[89,15414,15415],{"encoding":91},"278528N",[29,15417,15419],{"className":15418,"ariaHidden":61},[96],[29,15420,15422,15425,15428],{"className":15421},[100],[29,15423],{"className":15424,"style":3718},[104],[29,15426,15410],{"className":15427},[109],[29,15429,3534],{"className":15430,"style":3667},[109,113],[14668,15432,15433],{},[14,15434,15435],{},"4.25 : 1",[10,15437,15438],{},"用 roofline 分析一下。H100 SXM 的 BF16 算力约 989 TFLOPS，HBM3 带宽 3.35 TB\u002Fs，拐点在：",[29,15440,15442],{"className":15441},[597],[29,15443,15445,15483],{"className":15444},[32],[29,15446,15448],{"className":15447},[36],[38,15449,15450],{"xmlns":40,"display":606},[42,15451,15452,15480],{},[45,15453,15454,15472,15474,15477],{},[1088,15455,15456,15464],{},[45,15457,15458,15461],{},[55,15459,15460],{},"989",[76,15462,15463],{}," TFLOPS",[45,15465,15466,15469],{},[55,15467,15468],{},"3.35",[76,15470,15471],{}," TB\u002Fs",[59,15473,3843],{},[55,15475,15476],{},"295",[76,15478,15479],{}," FLOP\u002Fbyte",[89,15481,15482],{"encoding":91},"\\frac{989\\ \\text{TFLOPS}}{3.35\\ \\text{TB\u002Fs}} \\approx 295\\ \\text{FLOP\u002Fbyte}",[29,15484,15486,15586],{"className":15485,"ariaHidden":61},[96],[29,15487,15489,15493,15577,15580,15583],{"className":15488},[100],[29,15490],{"className":15491,"style":15492},[104],"height:2.2963em;vertical-align:-0.936em;",[29,15494,15496,15499,15574],{"className":15495},[109],[29,15497],{"className":15498},[483,1549],[29,15500,15502],{"className":15501},[1088],[29,15503,15505,15565],{"className":15504},[121,122],[29,15506,15508,15562],{"className":15507},[126],[29,15509,15512,15533,15541],{"className":15510,"style":15511},[130],"height:1.3603em;",[29,15513,15514,15517],{"style":5413},[29,15515],{"className":15516,"style":5417},[138],[29,15518,15520,15523,15526],{"className":15519},[109],[29,15521,15468],{"className":15522},[109],[29,15524,5624],{"className":15525},[171],[29,15527,15529],{"className":15528},[109,3686],[29,15530,15532],{"className":15531},[109],"TB\u002Fs",[29,15534,15535,15538],{"style":5426},[29,15536],{"className":15537,"style":5417},[138],[29,15539],{"className":15540,"style":1776},[1775],[29,15542,15543,15546],{"style":5435},[29,15544],{"className":15545,"style":5417},[138],[29,15547,15549,15552,15555],{"className":15548},[109],[29,15550,15460],{"className":15551},[109],[29,15553,5624],{"className":15554},[171],[29,15556,15558],{"className":15557},[109,3686],[29,15559,15561],{"className":15560},[109],"TFLOPS",[29,15563,154],{"className":15564},[153],[29,15566,15568],{"className":15567},[126],[29,15569,15572],{"className":15570,"style":15571},[130],"height:0.936em;",[29,15573],{},[29,15575],{"className":15576},[490,1549],[29,15578],{"className":15579,"style":367},[171],[29,15581,3843],{"className":15582},[371],[29,15584],{"className":15585,"style":367},[171],[29,15587,15589,15592,15595,15598],{"className":15588},[100],[29,15590],{"className":15591,"style":479},[104],[29,15593,15476],{"className":15594},[109],[29,15596,5624],{"className":15597},[171],[29,15599,15601],{"className":15600},[109,3686],[29,15602,15604],{"className":15603},[109],"FLOP\u002Fbyte",[10,15606,15607],{},"算术强度低于 295 就是 memory bound，高于才是 compute bound。再看 MLA decode 这一侧，别忘了每 token 还要把权重读一遍。",[15609,15610,15611,15643,15696,15984,16306],"ul",{},[15612,15613,15614,15642],"li",{},[29,15615,15617,15630],{"className":15616},[32],[29,15618,15620],{"className":15619},[36],[38,15621,15622],{"xmlns":40},[42,15623,15624,15628],{},[45,15625,15626],{},[51,15627,3534],{},[89,15629,3534],{"encoding":91},[29,15631,15633],{"className":15632,"ariaHidden":61},[96],[29,15634,15636,15639],{"className":15635},[100],[29,15637],{"className":15638,"style":3718},[104],[29,15640,3534],{"className":15641,"style":3667},[109,113],"：当前 context 长度，也就是 decode 到这一步时要看多少个历史 token",[15612,15644,15645,15695],{},[29,15646,15648,15665],{"className":15647},[32],[29,15649,15651],{"className":15650},[36],[38,15652,15653],{"xmlns":40},[42,15654,15655,15663],{},[45,15656,15657,15659,15661],{},[51,15658,3517],{},[59,15660,508],{},[55,15662,14198],{},[89,15664,14201],{"encoding":91},[29,15666,15668,15686],{"className":15667,"ariaHidden":61},[96],[29,15669,15671,15674,15677,15680,15683],{"className":15670},[100],[29,15672],{"className":15673,"style":3718},[104],[29,15675,3517],{"className":15676},[109,113],[29,15678],{"className":15679,"style":367},[171],[29,15681,508],{"className":15682},[371],[29,15684],{"className":15685,"style":367},[171],[29,15687,15689,15692],{"className":15688},[100],[29,15690],{"className":15691,"style":3272},[104],[29,15693,14198],{"className":15694},[109],"：层数",[15612,15697,15698,15983],{},[29,15699,15701,15755],{"className":15700},[32],[29,15702,15704],{"className":15703},[36],[38,15705,15706],{"xmlns":40},[42,15707,15708,15752],{},[45,15709,15710,15717,15719,15721,15727,15729,15735,15737,15739,15741,15744,15746,15748,15750],{},[48,15711,15712,15714],{},[51,15713,6467],{},[76,15715,15716],{},"kv",[59,15718,508],{},[59,15720,460],{"stretchy":459},[48,15722,15723,15725],{},[51,15724,309],{},[51,15726,6467],{},[59,15728,3236],{},[48,15730,15731,15733],{},[51,15732,309],{},[51,15734,13458],{},[59,15736,466],{"stretchy":459},[51,15738,14576],{},[59,15740,508],{},[55,15742,15743],{},"576",[59,15745,3514],{},[55,15747,69],{},[59,15749,508],{},[55,15751,15328],{},[89,15753,15754],{"encoding":91},"c_{\\text{kv}} = (d_c + d_r)b = 576 \\times 2 = 1152",[29,15756,15758,15819,15877,15938,15956,15974],{"className":15757,"ariaHidden":61},[96],[29,15759,15761,15764,15810,15813,15816],{"className":15760},[100],[29,15762],{"className":15763,"style":692},[104],[29,15765,15767,15770],{"className":15766},[109],[29,15768,6467],{"className":15769},[109,113],[29,15771,15773],{"className":15772},[117],[29,15774,15776,15802],{"className":15775},[121,122],[29,15777,15779,15799],{"className":15778},[126],[29,15780,15782],{"className":15781,"style":253},[130],[29,15783,15784,15787],{"style":134},[29,15785],{"className":15786,"style":139},[138],[29,15788,15790],{"className":15789},[143,144,145,146],[29,15791,15793],{"className":15792},[109,146],[29,15794,15796],{"className":15795},[109,3686,146],[29,15797,15716],{"className":15798},[109,146],[29,15800,154],{"className":15801},[153],[29,15803,15805],{"className":15804},[126],[29,15806,15808],{"className":15807,"style":161},[130],[29,15809],{},[29,15811],{"className":15812,"style":367},[171],[29,15814,508],{"className":15815},[371],[29,15817],{"className":15818,"style":367},[171],[29,15820,15822,15825,15828,15868,15871,15874],{"className":15821},[100],[29,15823],{"className":15824,"style":479},[104],[29,15826,460],{"className":15827},[483],[29,15829,15831,15834],{"className":15830},[109],[29,15832,309],{"className":15833},[109,113],[29,15835,15837],{"className":15836},[117],[29,15838,15840,15860],{"className":15839},[121,122],[29,15841,15843,15857],{"className":15842},[126],[29,15844,15846],{"className":15845,"style":3172},[130],[29,15847,15848,15851],{"style":134},[29,15849],{"className":15850,"style":139},[138],[29,15852,15854],{"className":15853},[143,144,145,146],[29,15855,6467],{"className":15856},[109,113,146],[29,15858,154],{"className":15859},[153],[29,15861,15863],{"className":15862},[126],[29,15864,15866],{"className":15865,"style":161},[130],[29,15867],{},[29,15869],{"className":15870,"style":3258},[171],[29,15872,3236],{"className":15873},[3262],[29,15875],{"className":15876,"style":3258},[171],[29,15878,15880,15883,15923,15926,15929,15932,15935],{"className":15879},[100],[29,15881],{"className":15882,"style":479},[104],[29,15884,15886,15889],{"className":15885},[109],[29,15887,309],{"className":15888},[109,113],[29,15890,15892],{"className":15891},[117],[29,15893,15895,15915],{"className":15894},[121,122],[29,15896,15898,15912],{"className":15897},[126],[29,15899,15901],{"className":15900,"style":3172},[130],[29,15902,15903,15906],{"style":134},[29,15904],{"className":15905,"style":139},[138],[29,15907,15909],{"className":15908},[143,144,145,146],[29,15910,13458],{"className":15911,"style":4465},[109,113,146],[29,15913,154],{"className":15914},[153],[29,15916,15918],{"className":15917},[126],[29,15919,15921],{"className":15920,"style":161},[130],[29,15922],{},[29,15924,466],{"className":15925},[490],[29,15927,14576],{"className":15928},[109,113],[29,15930],{"className":15931,"style":367},[171],[29,15933,508],{"className":15934},[371],[29,15936],{"className":15937,"style":367},[171],[29,15939,15941,15944,15947,15950,15953],{"className":15940},[100],[29,15942],{"className":15943,"style":3552},[104],[29,15945,15743],{"className":15946},[109],[29,15948],{"className":15949,"style":3258},[171],[29,15951,3514],{"className":15952},[3262],[29,15954],{"className":15955,"style":3258},[171],[29,15957,15959,15962,15965,15968,15971],{"className":15958},[100],[29,15960],{"className":15961,"style":3272},[104],[29,15963,69],{"className":15964},[109],[29,15966],{"className":15967,"style":367},[171],[29,15969,508],{"className":15970},[371],[29,15972],{"className":15973,"style":367},[171],[29,15975,15977,15980],{"className":15976},[100],[29,15978],{"className":15979,"style":3272},[104],[29,15981,15328],{"className":15982},[109]," 字节：每层、每 token、每多看一个历史位置，要多读多少 KV",[15612,15985,15986,16305],{},[29,15987,15989,16048],{"className":15988},[32],[29,15990,15992],{"className":15991},[36],[38,15993,15994],{"xmlns":40},[42,15995,15996,16045],{},[45,15997,15998,16005,16007,16009,16011,16013,16019,16021,16027,16029,16031,16033,16035,16041,16043],{},[48,15999,16000,16002],{},[51,16001,6467],{},[76,16003,16004],{},"flop",[59,16006,508],{},[55,16008,69],{},[51,16010,428],{},[59,16012,460],{"stretchy":459},[48,16014,16015,16017],{},[51,16016,309],{},[51,16018,6467],{},[59,16020,3236],{},[48,16022,16023,16025],{},[51,16024,309],{},[51,16026,13458],{},[59,16028,466],{"stretchy":459},[59,16030,3236],{},[55,16032,69],{},[51,16034,428],{},[48,16036,16037,16039],{},[51,16038,309],{},[51,16040,6467],{},[59,16042,508],{},[55,16044,15410],{},[89,16046,16047],{"encoding":91},"c_{\\text{flop}} = 2h(d_c + d_r) + 2hd_c = 278528",[29,16049,16051,16113,16177,16235,16296],{"className":16050,"ariaHidden":61},[96],[29,16052,16054,16058,16104,16107,16110],{"className":16053},[100],[29,16055],{"className":16056,"style":16057},[104],"height:0.7167em;vertical-align:-0.2861em;",[29,16059,16061,16064],{"className":16060},[109],[29,16062,6467],{"className":16063},[109,113],[29,16065,16067],{"className":16066},[117],[29,16068,16070,16096],{"className":16069},[121,122],[29,16071,16073,16093],{"className":16072},[126],[29,16074,16076],{"className":16075,"style":253},[130],[29,16077,16078,16081],{"style":134},[29,16079],{"className":16080,"style":139},[138],[29,16082,16084],{"className":16083},[143,144,145,146],[29,16085,16087],{"className":16086},[109,146],[29,16088,16090],{"className":16089},[109,3686,146],[29,16091,16004],{"className":16092},[109,146],[29,16094,154],{"className":16095},[153],[29,16097,16099],{"className":16098},[126],[29,16100,16102],{"className":16101,"style":4281},[130],[29,16103],{},[29,16105],{"className":16106,"style":367},[171],[29,16108,508],{"className":16109},[371],[29,16111],{"className":16112,"style":367},[171],[29,16114,16116,16119,16122,16125,16128,16168,16171,16174],{"className":16115},[100],[29,16117],{"className":16118,"style":479},[104],[29,16120,69],{"className":16121},[109],[29,16123,428],{"className":16124},[109,113],[29,16126,460],{"className":16127},[483],[29,16129,16131,16134],{"className":16130},[109],[29,16132,309],{"className":16133},[109,113],[29,16135,16137],{"className":16136},[117],[29,16138,16140,16160],{"className":16139},[121,122],[29,16141,16143,16157],{"className":16142},[126],[29,16144,16146],{"className":16145,"style":3172},[130],[29,16147,16148,16151],{"style":134},[29,16149],{"className":16150,"style":139},[138],[29,16152,16154],{"className":16153},[143,144,145,146],[29,16155,6467],{"className":16156},[109,113,146],[29,16158,154],{"className":16159},[153],[29,16161,16163],{"className":16162},[126],[29,16164,16166],{"className":16165,"style":161},[130],[29,16167],{},[29,16169],{"className":16170,"style":3258},[171],[29,16172,3236],{"className":16173},[3262],[29,16175],{"className":16176,"style":3258},[171],[29,16178,16180,16183,16223,16226,16229,16232],{"className":16179},[100],[29,16181],{"className":16182,"style":479},[104],[29,16184,16186,16189],{"className":16185},[109],[29,16187,309],{"className":16188},[109,113],[29,16190,16192],{"className":16191},[117],[29,16193,16195,16215],{"className":16194},[121,122],[29,16196,16198,16212],{"className":16197},[126],[29,16199,16201],{"className":16200,"style":3172},[130],[29,16202,16203,16206],{"style":134},[29,16204],{"className":16205,"style":139},[138],[29,16207,16209],{"className":16208},[143,144,145,146],[29,16210,13458],{"className":16211,"style":4465},[109,113,146],[29,16213,154],{"className":16214},[153],[29,16216,16218],{"className":16217},[126],[29,16219,16221],{"className":16220,"style":161},[130],[29,16222],{},[29,16224,466],{"className":16225},[490],[29,16227],{"className":16228,"style":3258},[171],[29,16230,3236],{"className":16231},[3262],[29,16233],{"className":16234,"style":3258},[171],[29,16236,16238,16241,16244,16247,16287,16290,16293],{"className":16237},[100],[29,16239],{"className":16240,"style":3098},[104],[29,16242,69],{"className":16243},[109],[29,16245,428],{"className":16246},[109,113],[29,16248,16250,16253],{"className":16249},[109],[29,16251,309],{"className":16252},[109,113],[29,16254,16256],{"className":16255},[117],[29,16257,16259,16279],{"className":16258},[121,122],[29,16260,16262,16276],{"className":16261},[126],[29,16263,16265],{"className":16264,"style":3172},[130],[29,16266,16267,16270],{"style":134},[29,16268],{"className":16269,"style":139},[138],[29,16271,16273],{"className":16272},[143,144,145,146],[29,16274,6467],{"className":16275},[109,113,146],[29,16277,154],{"className":16278},[153],[29,16280,16282],{"className":16281},[126],[29,16283,16285],{"className":16284,"style":161},[130],[29,16286],{},[29,16288],{"className":16289,"style":367},[171],[29,16291,508],{"className":16292},[371],[29,16294],{"className":16295,"style":367},[171],[29,16297,16299,16302],{"className":16298},[100],[29,16300],{"className":16301,"style":3272},[104],[29,16303,15410],{"className":16304},[109],"：每层、每 token、每多看一个历史位置，要多做多少次浮点运算",[15612,16307,16308,16427,16428,16456],{},[29,16309,16311,16344],{"className":16310},[32],[29,16312,16314],{"className":16313},[36],[38,16315,16316],{"xmlns":40},[42,16317,16318,16341],{},[45,16319,16320,16322,16324,16327,16329,16331,16333,16336,16338],{},[51,16321,2087],{},[59,16323,508],{},[55,16325,16326],{},"21",[76,16328,15311],{},[59,16330,3514],{},[55,16332,69],{},[76,16334,16335],{}," 字节",[59,16337,508],{},[55,16339,16340],{},"42",[89,16342,16343],{"encoding":91},"W = 21\\text{B} \\times 2\\ \\text{字节} = 42",[29,16345,16347,16365,16389,16418],{"className":16346,"ariaHidden":61},[96],[29,16348,16350,16353,16356,16359,16362],{"className":16349},[100],[29,16351],{"className":16352,"style":3718},[104],[29,16354,2087],{"className":16355,"style":2350},[109,113],[29,16357],{"className":16358,"style":367},[171],[29,16360,508],{"className":16361},[371],[29,16363],{"className":16364,"style":367},[171],[29,16366,16368,16371,16374,16380,16383,16386],{"className":16367},[100],[29,16369],{"className":16370,"style":3571},[104],[29,16372,16326],{"className":16373},[109],[29,16375,16377],{"className":16376},[109,3686],[29,16378,15311],{"className":16379},[109],[29,16381],{"className":16382,"style":3258},[171],[29,16384,3514],{"className":16385},[3262],[29,16387],{"className":16388,"style":3258},[171],[29,16390,16392,16395,16398,16401,16409,16412,16415],{"className":16391},[100],[29,16393],{"className":16394,"style":3718},[104],[29,16396,69],{"className":16397},[109],[29,16399,5624],{"className":16400},[171],[29,16402,16404],{"className":16403},[109,3686],[29,16405,16408],{"className":16406},[109,16407],"cjk_fallback","字节",[29,16410],{"className":16411,"style":367},[171],[29,16413,508],{"className":16414},[371],[29,16416],{"className":16417,"style":367},[171],[29,16419,16421,16424],{"className":16420},[100],[29,16422],{"className":16423,"style":3272},[104],[29,16425,16340],{"className":16426},[109]," GB：每 token 要读的权重字节数（21B 是 activated 参数量），它不随 ",[29,16429,16431,16444],{"className":16430},[32],[29,16432,16434],{"className":16433},[36],[38,16435,16436],{"xmlns":40},[42,16437,16438,16442],{},[45,16439,16440],{},[51,16441,3534],{},[89,16443,3534],{"encoding":91},[29,16445,16447],{"className":16446,"ariaHidden":61},[96],[29,16448,16450,16453],{"className":16449},[100],[29,16451],{"className":16452,"style":3718},[104],[29,16454,3534],{"className":16455,"style":3667},[109,113]," 变",[10,16458,16459,16460,16490],{},"算术强度（记作 ",[29,16461,16463,16477],{"className":16462},[32],[29,16464,16466],{"className":16465},[36],[38,16467,16468],{"xmlns":40},[42,16469,16470,16475],{},[45,16471,16472],{},[51,16473,16474],{},"I",[89,16476,16474],{"encoding":91},[29,16478,16480],{"className":16479,"ariaHidden":61},[96],[29,16481,16483,16486],{"className":16482},[100],[29,16484],{"className":16485,"style":3718},[104],[29,16487,16474],{"className":16488,"style":16489},[109,113],"margin-right:0.0785em;","）就是一个字节要承担多少次浮点运算：",[29,16492,16494],{"className":16493},[597],[29,16495,16497,16565],{"className":16496},[32],[29,16498,16500],{"className":16499},[36],[38,16501,16502],{"xmlns":40,"display":606},[42,16503,16504,16562],{},[45,16505,16506,16508,16510,16512,16514,16516],{},[51,16507,16474],{},[59,16509,460],{"stretchy":459},[51,16511,3534],{},[59,16513,466],{"stretchy":459},[59,16515,508],{},[1088,16517,16518,16542],{},[45,16519,16520,16522,16525,16527,16533,16536,16538,16540],{},[55,16521,69],{},[51,16523,16524],{},"P",[59,16526,3236],{},[48,16528,16529,16531],{},[51,16530,6467],{},[76,16532,16004],{},[59,16534,16535],{},"⋅",[51,16537,3517],{},[59,16539,16535],{},[51,16541,3534],{},[45,16543,16544,16546,16548,16554,16556,16558,16560],{},[51,16545,2087],{},[59,16547,3236],{},[48,16549,16550,16552],{},[51,16551,6467],{},[76,16553,15716],{},[59,16555,16535],{},[51,16557,3517],{},[59,16559,16535],{},[51,16561,3534],{},[89,16563,16564],{"encoding":91},"I(N) = \\frac{2P + c_{\\text{flop}} \\cdot L \\cdot N}{W + c_{\\text{kv}} \\cdot L \\cdot N}",[29,16566,16568,16595],{"className":16567,"ariaHidden":61},[96],[29,16569,16571,16574,16577,16580,16583,16586,16589,16592],{"className":16570},[100],[29,16572],{"className":16573,"style":479},[104],[29,16575,16474],{"className":16576,"style":16489},[109,113],[29,16578,460],{"className":16579},[483],[29,16581,3534],{"className":16582,"style":3667},[109,113],[29,16584,466],{"className":16585},[490],[29,16587],{"className":16588,"style":367},[171],[29,16590,508],{"className":16591},[371],[29,16593],{"className":16594,"style":367},[171],[29,16596,16598,16602],{"className":16597},[100],[29,16599],{"className":16600,"style":16601},[104],"height:2.1963em;vertical-align:-0.836em;",[29,16603,16605,16608,16823],{"className":16604},[109],[29,16606],{"className":16607},[483,1549],[29,16609,16611],{"className":16610},[1088],[29,16612,16614,16814],{"className":16613},[121,122],[29,16615,16617,16811],{"className":16616},[126],[29,16618,16620,16710,16718],{"className":16619,"style":15511},[130],[29,16621,16622,16625],{"style":5413},[29,16623],{"className":16624,"style":5417},[138],[29,16626,16628,16631,16634,16637,16640,16686,16689,16692,16695,16698,16701,16704,16707],{"className":16627},[109],[29,16629,2087],{"className":16630,"style":2350},[109,113],[29,16632],{"className":16633,"style":3258},[171],[29,16635,3236],{"className":16636},[3262],[29,16638],{"className":16639,"style":3258},[171],[29,16641,16643,16646],{"className":16642},[109],[29,16644,6467],{"className":16645},[109,113],[29,16647,16649],{"className":16648},[117],[29,16650,16652,16678],{"className":16651},[121,122],[29,16653,16655,16675],{"className":16654},[126],[29,16656,16658],{"className":16657,"style":253},[130],[29,16659,16660,16663],{"style":134},[29,16661],{"className":16662,"style":139},[138],[29,16664,16666],{"className":16665},[143,144,145,146],[29,16667,16669],{"className":16668},[109,146],[29,16670,16672],{"className":16671},[109,3686,146],[29,16673,15716],{"className":16674},[109,146],[29,16676,154],{"className":16677},[153],[29,16679,16681],{"className":16680},[126],[29,16682,16684],{"className":16683,"style":161},[130],[29,16685],{},[29,16687],{"className":16688,"style":3258},[171],[29,16690,16535],{"className":16691},[3262],[29,16693],{"className":16694,"style":3258},[171],[29,16696,3517],{"className":16697},[109,113],[29,16699],{"className":16700,"style":3258},[171],[29,16702,16535],{"className":16703},[3262],[29,16705],{"className":16706,"style":3258},[171],[29,16708,3534],{"className":16709,"style":3667},[109,113],[29,16711,16712,16715],{"style":5426},[29,16713],{"className":16714,"style":5417},[138],[29,16716],{"className":16717,"style":1776},[1775],[29,16719,16720,16723],{"style":5435},[29,16721],{"className":16722,"style":5417},[138],[29,16724,16726,16729,16732,16735,16738,16741,16787,16790,16793,16796,16799,16802,16805,16808],{"className":16725},[109],[29,16727,69],{"className":16728},[109],[29,16730,16524],{"className":16731,"style":2350},[109,113],[29,16733],{"className":16734,"style":3258},[171],[29,16736,3236],{"className":16737},[3262],[29,16739],{"className":16740,"style":3258},[171],[29,16742,16744,16747],{"className":16743},[109],[29,16745,6467],{"className":16746},[109,113],[29,16748,16750],{"className":16749},[117],[29,16751,16753,16779],{"className":16752},[121,122],[29,16754,16756,16776],{"className":16755},[126],[29,16757,16759],{"className":16758,"style":253},[130],[29,16760,16761,16764],{"style":134},[29,16762],{"className":16763,"style":139},[138],[29,16765,16767],{"className":16766},[143,144,145,146],[29,16768,16770],{"className":16769},[109,146],[29,16771,16773],{"className":16772},[109,3686,146],[29,16774,16004],{"className":16775},[109,146],[29,16777,154],{"className":16778},[153],[29,16780,16782],{"className":16781},[126],[29,16783,16785],{"className":16784,"style":4281},[130],[29,16786],{},[29,16788],{"className":16789,"style":3258},[171],[29,16791,16535],{"className":16792},[3262],[29,16794],{"className":16795,"style":3258},[171],[29,16797,3517],{"className":16798},[109,113],[29,16800],{"className":16801,"style":3258},[171],[29,16803,16535],{"className":16804},[3262],[29,16806],{"className":16807,"style":3258},[171],[29,16809,3534],{"className":16810,"style":3667},[109,113],[29,16812,154],{"className":16813},[153],[29,16815,16817],{"className":16816},[126],[29,16818,16821],{"className":16819,"style":16820},[130],"height:0.836em;",[29,16822],{},[29,16824],{"className":16825},[490,1549],[10,16827,16828,16829,16863,16864,16892],{},"分子是每个 token 要做的浮点次数：权重矩阵乘 ",[29,16830,16832,16848],{"className":16831},[32],[29,16833,16835],{"className":16834},[36],[38,16836,16837],{"xmlns":40},[42,16838,16839,16845],{},[45,16840,16841,16843],{},[55,16842,69],{},[51,16844,16524],{},[89,16846,16847],{"encoding":91},"2P",[29,16849,16851],{"className":16850,"ariaHidden":61},[96],[29,16852,16854,16857,16860],{"className":16853},[100],[29,16855],{"className":16856,"style":3718},[104],[29,16858,69],{"className":16859},[109],[29,16861,16524],{"className":16862,"style":2350},[109,113],"，加上 attention 那部分。分母是要搬的字节：权重 ",[29,16865,16867,16880],{"className":16866},[32],[29,16868,16870],{"className":16869},[36],[38,16871,16872],{"xmlns":40},[42,16873,16874,16878],{},[45,16875,16876],{},[51,16877,2087],{},[89,16879,2087],{"encoding":91},[29,16881,16883],{"className":16882,"ariaHidden":61},[96],[29,16884,16886,16889],{"className":16885},[100],[29,16887],{"className":16888,"style":3718},[104],[29,16890,2087],{"className":16891,"style":2350},[109,113],"，加上 KV。代进数字：",[29,16894,16896],{"className":16895},[597],[29,16897,16899,16973],{"className":16898},[32],[29,16900,16902],{"className":16901},[36],[38,16903,16904],{"xmlns":40,"display":606},[42,16905,16906,16970],{},[45,16907,16908,16910,16912,16914,16916,16918],{},[51,16909,16474],{},[59,16911,460],{"stretchy":459},[51,16913,3534],{},[59,16915,466],{"stretchy":459},[59,16917,508],{},[1088,16919,16920,16946],{},[45,16921,16922,16924,16926,16934,16936,16938,16940,16942,16944],{},[55,16923,16340],{},[59,16925,3514],{},[301,16927,16928,16931],{},[55,16929,16930],{},"10",[55,16932,16933],{},"9",[59,16935,3236],{},[55,16937,15410],{},[59,16939,3514],{},[55,16941,14198],{},[59,16943,3514],{},[51,16945,3534],{},[45,16947,16948,16950,16952,16958,16960,16962,16964,16966,16968],{},[55,16949,16340],{},[59,16951,3514],{},[301,16953,16954,16956],{},[55,16955,16930],{},[55,16957,16933],{},[59,16959,3236],{},[55,16961,15328],{},[59,16963,3514],{},[55,16965,14198],{},[59,16967,3514],{},[51,16969,3534],{},[89,16971,16972],{"encoding":91},"I(N) = \\frac{42 \\times 10^9 + 278528 \\times 60 \\times N}{42 \\times 10^9 + 1152 \\times 60 \\times N}",[29,16974,16976,17003],{"className":16975,"ariaHidden":61},[96],[29,16977,16979,16982,16985,16988,16991,16994,16997,17000],{"className":16978},[100],[29,16980],{"className":16981,"style":479},[104],[29,16983,16474],{"className":16984,"style":16489},[109,113],[29,16986,460],{"className":16987},[483],[29,16989,3534],{"className":16990,"style":3667},[109,113],[29,16992,466],{"className":16993},[490],[29,16995],{"className":16996,"style":367},[171],[29,16998,508],{"className":16999},[371],[29,17001],{"className":17002,"style":367},[171],[29,17004,17006,17010],{"className":17005},[100],[29,17007],{"className":17008,"style":17009},[104],"height:2.2604em;vertical-align:-0.7693em;",[29,17011,17013,17016,17228],{"className":17012},[109],[29,17014],{"className":17015},[483,1549],[29,17017,17019],{"className":17018},[1088],[29,17020,17022,17219],{"className":17021},[121,122],[29,17023,17025,17216],{"className":17024},[126],[29,17026,17029,17120,17128],{"className":17027,"style":17028},[130],"height:1.4911em;",[29,17030,17031,17034],{"style":5413},[29,17032],{"className":17033,"style":5417},[138],[29,17035,17037,17040,17043,17046,17049,17052,17084,17087,17090,17093,17096,17099,17102,17105,17108,17111,17114,17117],{"className":17036},[109],[29,17038,16340],{"className":17039},[109],[29,17041],{"className":17042,"style":3258},[171],[29,17044,3514],{"className":17045},[3262],[29,17047],{"className":17048,"style":3258},[171],[29,17050,57],{"className":17051},[109],[29,17053,17055,17059],{"className":17054},[109],[29,17056,17058],{"className":17057},[109],"0",[29,17060,17062],{"className":17061},[117],[29,17063,17065],{"className":17064},[121],[29,17066,17068],{"className":17067},[126],[29,17069,17072],{"className":17070,"style":17071},[130],"height:0.7401em;",[29,17073,17075,17078],{"style":17074},"top:-2.989em;margin-right:0.05em;",[29,17076],{"className":17077,"style":139},[138],[29,17079,17081],{"className":17080},[143,144,145,146],[29,17082,16933],{"className":17083},[109,146],[29,17085],{"className":17086,"style":3258},[171],[29,17088,3236],{"className":17089},[3262],[29,17091],{"className":17092,"style":3258},[171],[29,17094,15328],{"className":17095},[109],[29,17097],{"className":17098,"style":3258},[171],[29,17100,3514],{"className":17101},[3262],[29,17103],{"className":17104,"style":3258},[171],[29,17106,14198],{"className":17107},[109],[29,17109],{"className":17110,"style":3258},[171],[29,17112,3514],{"className":17113},[3262],[29,17115],{"className":17116,"style":3258},[171],[29,17118,3534],{"className":17119,"style":3667},[109,113],[29,17121,17122,17125],{"style":5426},[29,17123],{"className":17124,"style":5417},[138],[29,17126],{"className":17127,"style":1776},[1775],[29,17129,17130,17133],{"style":5435},[29,17131],{"className":17132,"style":5417},[138],[29,17134,17136,17139,17142,17145,17148,17151,17180,17183,17186,17189,17192,17195,17198,17201,17204,17207,17210,17213],{"className":17135},[109],[29,17137,16340],{"className":17138},[109],[29,17140],{"className":17141,"style":3258},[171],[29,17143,3514],{"className":17144},[3262],[29,17146],{"className":17147,"style":3258},[171],[29,17149,57],{"className":17150},[109],[29,17152,17154,17157],{"className":17153},[109],[29,17155,17058],{"className":17156},[109],[29,17158,17160],{"className":17159},[117],[29,17161,17163],{"className":17162},[121],[29,17164,17166],{"className":17165},[126],[29,17167,17169],{"className":17168,"style":4490},[130],[29,17170,17171,17174],{"style":403},[29,17172],{"className":17173,"style":139},[138],[29,17175,17177],{"className":17176},[143,144,145,146],[29,17178,16933],{"className":17179},[109,146],[29,17181],{"className":17182,"style":3258},[171],[29,17184,3236],{"className":17185},[3262],[29,17187],{"className":17188,"style":3258},[171],[29,17190,15410],{"className":17191},[109],[29,17193],{"className":17194,"style":3258},[171],[29,17196,3514],{"className":17197},[3262],[29,17199],{"className":17200,"style":3258},[171],[29,17202,14198],{"className":17203},[109],[29,17205],{"className":17206,"style":3258},[171],[29,17208,3514],{"className":17209},[3262],[29,17211],{"className":17212,"style":3258},[171],[29,17214,3534],{"className":17215,"style":3667},[109,113],[29,17217,154],{"className":17218},[153],[29,17220,17222],{"className":17221},[126],[29,17223,17226],{"className":17224,"style":17225},[130],"height:0.7693em;",[29,17227],{},[29,17229],{"className":17230},[490,1549],[10,17232,17233,17261],{},[29,17234,17236,17249],{"className":17235},[32],[29,17237,17239],{"className":17238},[36],[38,17240,17241],{"xmlns":40},[42,17242,17243,17247],{},[45,17244,17245],{},[51,17246,3534],{},[89,17248,3534],{"encoding":91},[29,17250,17252],{"className":17251,"ariaHidden":61},[96],[29,17253,17255,17258],{"className":17254},[100],[29,17256],{"className":17257,"style":3718},[104],[29,17259,3534],{"className":17260,"style":3667},[109,113]," 一大，分子分母里那两项权重就被 KV 和 attention 项压过去了，于是有上界：",[29,17263,17265],{"className":17264},[597],[29,17266,17268,17337],{"className":17267},[32],[29,17269,17271],{"className":17270},[36],[38,17272,17273],{"xmlns":40,"display":606},[42,17274,17275,17334],{},[45,17276,17277,17295,17297,17299,17301,17303,17305,17319,17321,17327,17329,17332],{},[1093,17278,17279,17286],{},[45,17280,17281,17284],{},[51,17282,17283],{},"lim",[59,17285,1111],{},[45,17287,17288,17290,17292],{},[51,17289,3534],{},[59,17291,8389],{},[51,17293,17294],{"mathvariant":1002},"∞",[51,17296,16474],{},[59,17298,460],{"stretchy":459},[51,17300,3534],{},[59,17302,466],{"stretchy":459},[59,17304,508],{},[1088,17306,17307,17313],{},[48,17308,17309,17311],{},[51,17310,6467],{},[76,17312,16004],{},[48,17314,17315,17317],{},[51,17316,6467],{},[76,17318,15716],{},[59,17320,508],{},[1088,17322,17323,17325],{},[55,17324,15410],{},[55,17326,15328],{},[59,17328,3843],{},[55,17330,17331],{},"242",[76,17333,15479],{},[89,17335,17336],{"encoding":91},"\\lim_{N \\to \\infty} I(N) = \\frac{c_{\\text{flop}}}{c_{\\text{kv}}} = \\frac{278528}{1152} \\approx 242\\ \\text{FLOP\u002Fbyte}",[29,17338,17340,17428,17592,17671],{"className":17339,"ariaHidden":61},[96],[29,17341,17343,17347,17404,17407,17410,17413,17416,17419,17422,17425],{"className":17342},[100],[29,17344],{"className":17345,"style":17346},[104],"height:1.4943em;vertical-align:-0.7443em;",[29,17348,17351],{"className":17349},[1576,17350],"op-limits",[29,17352,17354,17395],{"className":17353},[121,122],[29,17355,17357,17392],{"className":17356},[126],[29,17358,17360,17381],{"className":17359,"style":440},[130],[29,17361,17363,17366],{"style":17362},"top:-2.3557em;margin-left:0em;",[29,17364],{"className":17365,"style":5417},[138],[29,17367,17369],{"className":17368},[143,144,145,146],[29,17370,17372,17375,17378],{"className":17371},[109,146],[29,17373,3534],{"className":17374,"style":3667},[109,113,146],[29,17376,8389],{"className":17377},[371,146],[29,17379,17294],{"className":17380},[109,146],[29,17382,17384,17387],{"style":17383},"top:-3em;",[29,17385],{"className":17386,"style":5417},[138],[29,17388,17389],{},[29,17390,17283],{"className":17391},[1576],[29,17393,154],{"className":17394},[153],[29,17396,17398],{"className":17397},[126],[29,17399,17402],{"className":17400,"style":17401},[130],"height:0.7443em;",[29,17403],{},[29,17405],{"className":17406,"style":172},[171],[29,17408,16474],{"className":17409,"style":16489},[109,113],[29,17411,460],{"className":17412},[483],[29,17414,3534],{"className":17415,"style":3667},[109,113],[29,17417,466],{"className":17418},[490],[29,17420],{"className":17421,"style":367},[171],[29,17423,508],{"className":17424},[371],[29,17426],{"className":17427,"style":367},[171],[29,17429,17431,17435,17583,17586,17589],{"className":17430},[100],[29,17432],{"className":17433,"style":17434},[104],"height:1.9436em;vertical-align:-0.836em;",[29,17436,17438,17441,17580],{"className":17437},[109],[29,17439],{"className":17440},[483,1549],[29,17442,17444],{"className":17443},[1088],[29,17445,17447,17572],{"className":17446},[121,122],[29,17448,17450,17569],{"className":17449},[126],[29,17451,17453,17507,17515],{"className":17452,"style":5410},[130],[29,17454,17455,17458],{"style":5413},[29,17456],{"className":17457,"style":5417},[138],[29,17459,17461],{"className":17460},[109],[29,17462,17464,17467],{"className":17463},[109],[29,17465,6467],{"className":17466},[109,113],[29,17468,17470],{"className":17469},[117],[29,17471,17473,17499],{"className":17472},[121,122],[29,17474,17476,17496],{"className":17475},[126],[29,17477,17479],{"className":17478,"style":253},[130],[29,17480,17481,17484],{"style":134},[29,17482],{"className":17483,"style":139},[138],[29,17485,17487],{"className":17486},[143,144,145,146],[29,17488,17490],{"className":17489},[109,146],[29,17491,17493],{"className":17492},[109,3686,146],[29,17494,15716],{"className":17495},[109,146],[29,17497,154],{"className":17498},[153],[29,17500,17502],{"className":17501},[126],[29,17503,17505],{"className":17504,"style":161},[130],[29,17506],{},[29,17508,17509,17512],{"style":5426},[29,17510],{"className":17511,"style":5417},[138],[29,17513],{"className":17514,"style":1776},[1775],[29,17516,17517,17520],{"style":5435},[29,17518],{"className":17519,"style":5417},[138],[29,17521,17523],{"className":17522},[109],[29,17524,17526,17529],{"className":17525},[109],[29,17527,6467],{"className":17528},[109,113],[29,17530,17532],{"className":17531},[117],[29,17533,17535,17561],{"className":17534},[121,122],[29,17536,17538,17558],{"className":17537},[126],[29,17539,17541],{"className":17540,"style":253},[130],[29,17542,17543,17546],{"style":134},[29,17544],{"className":17545,"style":139},[138],[29,17547,17549],{"className":17548},[143,144,145,146],[29,17550,17552],{"className":17551},[109,146],[29,17553,17555],{"className":17554},[109,3686,146],[29,17556,16004],{"className":17557},[109,146],[29,17559,154],{"className":17560},[153],[29,17562,17564],{"className":17563},[126],[29,17565,17567],{"className":17566,"style":4281},[130],[29,17568],{},[29,17570,154],{"className":17571},[153],[29,17573,17575],{"className":17574},[126],[29,17576,17578],{"className":17577,"style":16820},[130],[29,17579],{},[29,17581],{"className":17582},[490,1549],[29,17584],{"className":17585,"style":367},[171],[29,17587,508],{"className":17588},[371],[29,17590],{"className":17591,"style":367},[171],[29,17593,17595,17599,17662,17665,17668],{"className":17594},[100],[29,17596],{"className":17597,"style":17598},[104],"height:2.0074em;vertical-align:-0.686em;",[29,17600,17602,17605,17659],{"className":17601},[109],[29,17603],{"className":17604},[483,1549],[29,17606,17608],{"className":17607},[1088],[29,17609,17611,17651],{"className":17610},[121,122],[29,17612,17614,17648],{"className":17613},[126],[29,17615,17618,17629,17637],{"className":17616,"style":17617},[130],"height:1.3214em;",[29,17619,17620,17623],{"style":5413},[29,17621],{"className":17622,"style":5417},[138],[29,17624,17626],{"className":17625},[109],[29,17627,15328],{"className":17628},[109],[29,17630,17631,17634],{"style":5426},[29,17632],{"className":17633,"style":5417},[138],[29,17635],{"className":17636,"style":1776},[1775],[29,17638,17639,17642],{"style":5435},[29,17640],{"className":17641,"style":5417},[138],[29,17643,17645],{"className":17644},[109],[29,17646,15410],{"className":17647},[109],[29,17649,154],{"className":17650},[153],[29,17652,17654],{"className":17653},[126],[29,17655,17657],{"className":17656,"style":5457},[130],[29,17658],{},[29,17660],{"className":17661},[490,1549],[29,17663],{"className":17664,"style":367},[171],[29,17666,3843],{"className":17667},[371],[29,17669],{"className":17670,"style":367},[171],[29,17672,17674,17677,17680,17683],{"className":17673},[100],[29,17675],{"className":17676,"style":479},[104],[29,17678,17331],{"className":17679},[109],[29,17681,5624],{"className":17682},[171],[29,17684,17686],{"className":17685},[109,3686],[29,17687,15604],{"className":17688},[109],[10,17690,17691,17692,17720,17721,17724],{},"注意 ",[29,17693,17695,17708],{"className":17694},[32],[29,17696,17698],{"className":17697},[36],[38,17699,17700],{"xmlns":40},[42,17701,17702,17706],{},[45,17703,17704],{},[51,17705,3517],{},[89,17707,3517],{"encoding":91},[29,17709,17711],{"className":17710,"ariaHidden":61},[96],[29,17712,17714,17717],{"className":17713},[100],[29,17715],{"className":17716,"style":3718},[104],[29,17718,3517],{"className":17719},[109,113]," 被约掉了。",[14,17722,17723],{},"无论多少层，这个上限都一样","，因为 KV 和 attention 计算都随层数线性增长。",[10,17726,17727,17730],{},[14,17728,17729],{},"242 小于 295。"," 也就是说，不管 context 多长，MLA 的 decode 都落在拐点左边，是 memory bound。那 4.25 倍的计算量加进去就是白加的。算力本来就在空转等内存。",[10,17732,17733],{},"具体看几个长度：",[14645,17735,17736,17782],{},[14648,17737,17738],{},[14651,17739,17740,17770,17773,17776,17779],{},[14654,17741,17742],{},[29,17743,17745,17758],{"className":17744},[32],[29,17746,17748],{"className":17747},[36],[38,17749,17750],{"xmlns":40},[42,17751,17752,17756],{},[45,17753,17754],{},[51,17755,3534],{},[89,17757,3534],{"encoding":91},[29,17759,17761],{"className":17760,"ariaHidden":61},[96],[29,17762,17764,17767],{"className":17763},[100],[29,17765],{"className":17766,"style":3718},[104],[29,17768,3534],{"className":17769,"style":3667},[109,113],[14654,17771,17772],{},"算术强度",[14654,17774,17775],{},"内存耗时",[14654,17777,17778],{},"计算耗时",[14654,17780,17781],{},"瓶颈",[14663,17783,17784,17801,17817,17833],{},[14651,17785,17786,17789,17792,17795,17798],{},[14668,17787,17788],{},"4k",[14668,17790,17791],{},"2.6",[14668,17793,17794],{},"12.6 ms",[14668,17796,17797],{},"0.11 ms",[14668,17799,17800],{},"memory",[14651,17802,17803,17806,17809,17812,17815],{},[14668,17804,17805],{},"32k",[14668,17807,17808],{},"13.3",[14668,17810,17811],{},"13.2 ms",[14668,17813,17814],{},"0.60 ms",[14668,17816,17800],{},[14651,17818,17819,17822,17825,17828,17831],{},[14668,17820,17821],{},"128k",[14668,17823,17824],{},"43.7",[14668,17826,17827],{},"15.2 ms",[14668,17829,17830],{},"2.3 ms",[14668,17832,17800],{},[14651,17834,17835,17838,17841,17844,17847],{},[14668,17836,17837],{},"1M",[14668,17839,17840],{},"153",[14668,17842,17843],{},"34.2 ms",[14668,17845,17846],{},"17.8 ms",[14668,17848,17800],{},[10,17850,17851],{},"到 1M 的时候计算耗时（17.8 ms）已经追到内存耗时（34.2 ms）的一半了，但还没越过去。",[10,17853,17854],{},"不过这里只算了 batch size 等于 1。在线服务几乎不会单条跑，batch 是提高算术强度的另一条路子，而且它会改变结论。",[10,17856,17857,17858,17887],{},"原因很直接：权重每一步只需要读一次，然后同时喂给 batch 里所有序列；KV 读取却是一条序列一份。把 ",[29,17859,17861,17874],{"className":17860},[32],[29,17862,17864],{"className":17863},[36],[38,17865,17866],{"xmlns":40},[42,17867,17868,17872],{},[45,17869,17870],{},[51,17871,15311],{},[89,17873,15311],{"encoding":91},[29,17875,17877],{"className":17876,"ariaHidden":61},[96],[29,17878,17880,17883],{"className":17879},[100],[29,17881],{"className":17882,"style":3718},[104],[29,17884,15311],{"className":17885,"style":17886},[109,113],"margin-right:0.0502em;"," 写进去：",[29,17889,17891],{"className":17890},[597],[29,17892,17894,17968],{"className":17893},[32],[29,17895,17897],{"className":17896},[36],[38,17898,17899],{"xmlns":40,"display":606},[42,17900,17901,17965],{},[45,17902,17903,17905,17907,17909,17911,17913,17915,17917],{},[51,17904,16474],{},[59,17906,460],{"stretchy":459},[51,17908,15311],{},[59,17910,62],{"separator":61},[51,17912,3534],{},[59,17914,466],{"stretchy":459},[59,17916,508],{},[1088,17918,17919,17945],{},[45,17920,17921,17923],{},[51,17922,15311],{},[45,17924,17925,17927,17929,17931,17933,17939,17941,17943],{},[59,17926,460],{"fence":61},[55,17928,69],{},[51,17930,16524],{},[59,17932,3236],{},[48,17934,17935,17937],{},[51,17936,6467],{},[76,17938,16004],{},[51,17940,3517],{},[51,17942,3534],{},[59,17944,466],{"fence":61},[45,17946,17947,17949,17951,17953,17955,17961,17963],{},[51,17948,2087],{},[59,17950,3236],{},[51,17952,15311],{},[59,17954,16535],{},[48,17956,17957,17959],{},[51,17958,6467],{},[76,17960,15716],{},[51,17962,3517],{},[51,17964,3534],{},[89,17966,17967],{"encoding":91},"I(B, N) = \\frac{B\\left(2P + c_{\\text{flop}} L N\\right)}{W + B \\cdot c_{\\text{kv}} L N}",[29,17969,17971,18007],{"className":17970,"ariaHidden":61},[96],[29,17972,17974,17977,17980,17983,17986,17989,17992,17995,17998,18001,18004],{"className":17973},[100],[29,17975],{"className":17976,"style":479},[104],[29,17978,16474],{"className":17979,"style":16489},[109,113],[29,17981,460],{"className":17982},[483],[29,17984,15311],{"className":17985,"style":17886},[109,113],[29,17987,62],{"className":17988},[167],[29,17990],{"className":17991,"style":172},[171],[29,17993,3534],{"className":17994,"style":3667},[109,113],[29,17996,466],{"className":17997},[490],[29,17999],{"className":18000,"style":367},[171],[29,18002,508],{"className":18003},[371],[29,18005],{"className":18006,"style":367},[171],[29,18008,18010,18014],{"className":18009},[100],[29,18011],{"className":18012,"style":18013},[104],"height:2.263em;vertical-align:-0.836em;",[29,18015,18017,18020,18226],{"className":18016},[109],[29,18018],{"className":18019},[483,1549],[29,18021,18023],{"className":18022},[1088],[29,18024,18026,18218],{"className":18025},[121,122],[29,18027,18029,18215],{"className":18028},[126],[29,18030,18033,18117,18125],{"className":18031,"style":18032},[130],"height:1.427em;",[29,18034,18035,18038],{"style":5413},[29,18036],{"className":18037,"style":5417},[138],[29,18039,18041,18044,18047,18050,18053,18056,18059,18062,18065,18111,18114],{"className":18040},[109],[29,18042,2087],{"className":18043,"style":2350},[109,113],[29,18045],{"className":18046,"style":3258},[171],[29,18048,3236],{"className":18049},[3262],[29,18051],{"className":18052,"style":3258},[171],[29,18054,15311],{"className":18055,"style":17886},[109,113],[29,18057],{"className":18058,"style":3258},[171],[29,18060,16535],{"className":18061},[3262],[29,18063],{"className":18064,"style":3258},[171],[29,18066,18068,18071],{"className":18067},[109],[29,18069,6467],{"className":18070},[109,113],[29,18072,18074],{"className":18073},[117],[29,18075,18077,18103],{"className":18076},[121,122],[29,18078,18080,18100],{"className":18079},[126],[29,18081,18083],{"className":18082,"style":253},[130],[29,18084,18085,18088],{"style":134},[29,18086],{"className":18087,"style":139},[138],[29,18089,18091],{"className":18090},[143,144,145,146],[29,18092,18094],{"className":18093},[109,146],[29,18095,18097],{"className":18096},[109,3686,146],[29,18098,15716],{"className":18099},[109,146],[29,18101,154],{"className":18102},[153],[29,18104,18106],{"className":18105},[126],[29,18107,18109],{"className":18108,"style":161},[130],[29,18110],{},[29,18112,3517],{"className":18113},[109,113],[29,18115,3534],{"className":18116,"style":3667},[109,113],[29,18118,18119,18122],{"style":5426},[29,18120],{"className":18121,"style":5417},[138],[29,18123],{"className":18124,"style":1776},[1775],[29,18126,18127,18130],{"style":5435},[29,18128],{"className":18129,"style":5417},[138],[29,18131,18133,18136,18139],{"className":18132},[109],[29,18134,15311],{"className":18135,"style":17886},[109,113],[29,18137],{"className":18138,"style":172},[171],[29,18140,18142,18145,18148,18151,18154,18157,18160,18206,18209,18212],{"className":18141},[222],[29,18143,460],{"className":18144,"style":5386},[483,5385],[29,18146,69],{"className":18147},[109],[29,18149,16524],{"className":18150,"style":2350},[109,113],[29,18152],{"className":18153,"style":3258},[171],[29,18155,3236],{"className":18156},[3262],[29,18158],{"className":18159,"style":3258},[171],[29,18161,18163,18166],{"className":18162},[109],[29,18164,6467],{"className":18165},[109,113],[29,18167,18169],{"className":18168},[117],[29,18170,18172,18198],{"className":18171},[121,122],[29,18173,18175,18195],{"className":18174},[126],[29,18176,18178],{"className":18177,"style":253},[130],[29,18179,18180,18183],{"style":134},[29,18181],{"className":18182,"style":139},[138],[29,18184,18186],{"className":18185},[143,144,145,146],[29,18187,18189],{"className":18188},[109,146],[29,18190,18192],{"className":18191},[109,3686,146],[29,18193,16004],{"className":18194},[109,146],[29,18196,154],{"className":18197},[153],[29,18199,18201],{"className":18200},[126],[29,18202,18204],{"className":18203,"style":4281},[130],[29,18205],{},[29,18207,3517],{"className":18208},[109,113],[29,18210,3534],{"className":18211,"style":3667},[109,113],[29,18213,466],{"className":18214,"style":5386},[490,5385],[29,18216,154],{"className":18217},[153],[29,18219,18221],{"className":18220},[126],[29,18222,18224],{"className":18223,"style":16820},[130],[29,18225],{},[29,18227],{"className":18228},[490,1549],[10,18230,18231,18259,18260,18288],{},[29,18232,18234,18247],{"className":18233},[32],[29,18235,18237],{"className":18236},[36],[38,18238,18239],{"xmlns":40},[42,18240,18241,18245],{},[45,18242,18243],{},[51,18244,15311],{},[89,18246,15311],{"encoding":91},[29,18248,18250],{"className":18249,"ariaHidden":61},[96],[29,18251,18253,18256],{"className":18252},[100],[29,18254],{"className":18255,"style":3718},[104],[29,18257,15311],{"className":18258,"style":17886},[109,113]," 一大，分母里的 ",[29,18261,18263,18276],{"className":18262},[32],[29,18264,18266],{"className":18265},[36],[38,18267,18268],{"xmlns":40},[42,18269,18270,18274],{},[45,18271,18272],{},[51,18273,2087],{},[89,18275,2087],{"encoding":91},[29,18277,18279],{"className":18278,"ariaHidden":61},[96],[29,18280,18282,18285],{"className":18281},[100],[29,18283],{"className":18284,"style":3718},[104],[29,18286,2087],{"className":18287,"style":2350},[109,113]," 就被摊薄，算术强度趋近于：",[29,18290,18292],{"className":18291},[597],[29,18293,18295,18353],{"className":18294},[32],[29,18296,18298],{"className":18297},[36],[38,18299,18300],{"xmlns":40,"display":606},[42,18301,18302,18350],{},[45,18303,18304,18306,18308,18310,18312,18314,18316,18318],{},[51,18305,16474],{},[59,18307,460],{"stretchy":459},[51,18309,17294],{"mathvariant":1002},[59,18311,62],{"separator":61},[51,18313,3534],{},[59,18315,466],{"stretchy":459},[59,18317,508],{},[1088,18319,18320,18338],{},[45,18321,18322,18324,18326,18328,18334,18336],{},[55,18323,69],{},[51,18325,16524],{},[59,18327,3236],{},[48,18329,18330,18332],{},[51,18331,6467],{},[76,18333,16004],{},[51,18335,3517],{},[51,18337,3534],{},[45,18339,18340,18346,18348],{},[48,18341,18342,18344],{},[51,18343,6467],{},[76,18345,15716],{},[51,18347,3517],{},[51,18349,3534],{},[89,18351,18352],{"encoding":91},"I(\\infty, N) = \\frac{2P + c_{\\text{flop}} L N}{c_{\\text{kv}} L N}",[29,18354,18356,18392],{"className":18355,"ariaHidden":61},[96],[29,18357,18359,18362,18365,18368,18371,18374,18377,18380,18383,18386,18389],{"className":18358},[100],[29,18360],{"className":18361,"style":479},[104],[29,18363,16474],{"className":18364,"style":16489},[109,113],[29,18366,460],{"className":18367},[483],[29,18369,17294],{"className":18370},[109],[29,18372,62],{"className":18373},[167],[29,18375],{"className":18376,"style":172},[171],[29,18378,3534],{"className":18379,"style":3667},[109,113],[29,18381,466],{"className":18382},[490],[29,18384],{"className":18385,"style":367},[171],[29,18387,508],{"className":18388},[371],[29,18390],{"className":18391,"style":367},[171],[29,18393,18395,18398],{"className":18394},[100],[29,18396],{"className":18397,"style":16601},[104],[29,18399,18401,18404,18570],{"className":18400},[109],[29,18402],{"className":18403},[483,1549],[29,18405,18407],{"className":18406},[1088],[29,18408,18410,18562],{"className":18409},[121,122],[29,18411,18413,18559],{"className":18412},[126],[29,18414,18416,18476,18484],{"className":18415,"style":15511},[130],[29,18417,18418,18421],{"style":5413},[29,18419],{"className":18420,"style":5417},[138],[29,18422,18424,18470,18473],{"className":18423},[109],[29,18425,18427,18430],{"className":18426},[109],[29,18428,6467],{"className":18429},[109,113],[29,18431,18433],{"className":18432},[117],[29,18434,18436,18462],{"className":18435},[121,122],[29,18437,18439,18459],{"className":18438},[126],[29,18440,18442],{"className":18441,"style":253},[130],[29,18443,18444,18447],{"style":134},[29,18445],{"className":18446,"style":139},[138],[29,18448,18450],{"className":18449},[143,144,145,146],[29,18451,18453],{"className":18452},[109,146],[29,18454,18456],{"className":18455},[109,3686,146],[29,18457,15716],{"className":18458},[109,146],[29,18460,154],{"className":18461},[153],[29,18463,18465],{"className":18464},[126],[29,18466,18468],{"className":18467,"style":161},[130],[29,18469],{},[29,18471,3517],{"className":18472},[109,113],[29,18474,3534],{"className":18475,"style":3667},[109,113],[29,18477,18478,18481],{"style":5426},[29,18479],{"className":18480,"style":5417},[138],[29,18482],{"className":18483,"style":1776},[1775],[29,18485,18486,18489],{"style":5435},[29,18487],{"className":18488,"style":5417},[138],[29,18490,18492,18495,18498,18501,18504,18507,18553,18556],{"className":18491},[109],[29,18493,69],{"className":18494},[109],[29,18496,16524],{"className":18497,"style":2350},[109,113],[29,18499],{"className":18500,"style":3258},[171],[29,18502,3236],{"className":18503},[3262],[29,18505],{"className":18506,"style":3258},[171],[29,18508,18510,18513],{"className":18509},[109],[29,18511,6467],{"className":18512},[109,113],[29,18514,18516],{"className":18515},[117],[29,18517,18519,18545],{"className":18518},[121,122],[29,18520,18522,18542],{"className":18521},[126],[29,18523,18525],{"className":18524,"style":253},[130],[29,18526,18527,18530],{"style":134},[29,18528],{"className":18529,"style":139},[138],[29,18531,18533],{"className":18532},[143,144,145,146],[29,18534,18536],{"className":18535},[109,146],[29,18537,18539],{"className":18538},[109,3686,146],[29,18540,16004],{"className":18541},[109,146],[29,18543,154],{"className":18544},[153],[29,18546,18548],{"className":18547},[126],[29,18549,18551],{"className":18550,"style":4281},[130],[29,18552],{},[29,18554,3517],{"className":18555},[109,113],[29,18557,3534],{"className":18558,"style":3667},[109,113],[29,18560,154],{"className":18561},[153],[29,18563,18565],{"className":18564},[126],[29,18566,18568],{"className":18567,"style":16820},[130],[29,18569],{},[29,18571],{"className":18572},[490,1549],[10,18574,15251],{},[14645,18576,18577,18876],{},[14648,18578,18579],{},[14651,18580,18581,18611,18664,18717,18770,18823],{},[14654,18582,18583],{},[29,18584,18586,18599],{"className":18585},[32],[29,18587,18589],{"className":18588},[36],[38,18590,18591],{"xmlns":40},[42,18592,18593,18597],{},[45,18594,18595],{},[51,18596,3534],{},[89,18598,3534],{"encoding":91},[29,18600,18602],{"className":18601,"ariaHidden":61},[96],[29,18603,18605,18608],{"className":18604},[100],[29,18606],{"className":18607,"style":3718},[104],[29,18609,3534],{"className":18610,"style":3667},[109,113],[14654,18612,18613],{},[29,18614,18616,18634],{"className":18615},[32],[29,18617,18619],{"className":18618},[36],[38,18620,18621],{"xmlns":40},[42,18622,18623,18631],{},[45,18624,18625,18627,18629],{},[51,18626,15311],{},[59,18628,508],{},[55,18630,57],{},[89,18632,18633],{"encoding":91},"B=1",[29,18635,18637,18655],{"className":18636,"ariaHidden":61},[96],[29,18638,18640,18643,18646,18649,18652],{"className":18639},[100],[29,18641],{"className":18642,"style":3718},[104],[29,18644,15311],{"className":18645,"style":17886},[109,113],[29,18647],{"className":18648,"style":367},[171],[29,18650,508],{"className":18651},[371],[29,18653],{"className":18654,"style":367},[171],[29,18656,18658,18661],{"className":18657},[100],[29,18659],{"className":18660,"style":3272},[104],[29,18662,57],{"className":18663},[109],[14654,18665,18666],{},[29,18667,18669,18687],{"className":18668},[32],[29,18670,18672],{"className":18671},[36],[38,18673,18674],{"xmlns":40},[42,18675,18676,18684],{},[45,18677,18678,18680,18682],{},[51,18679,15311],{},[59,18681,508],{},[55,18683,6035],{},[89,18685,18686],{"encoding":91},"B=8",[29,18688,18690,18708],{"className":18689,"ariaHidden":61},[96],[29,18691,18693,18696,18699,18702,18705],{"className":18692},[100],[29,18694],{"className":18695,"style":3718},[104],[29,18697,15311],{"className":18698,"style":17886},[109,113],[29,18700],{"className":18701,"style":367},[171],[29,18703,508],{"className":18704},[371],[29,18706],{"className":18707,"style":367},[171],[29,18709,18711,18714],{"className":18710},[100],[29,18712],{"className":18713,"style":3272},[104],[29,18715,6035],{"className":18716},[109],[14654,18718,18719],{},[29,18720,18722,18740],{"className":18721},[32],[29,18723,18725],{"className":18724},[36],[38,18726,18727],{"xmlns":40},[42,18728,18729,18737],{},[45,18730,18731,18733,18735],{},[51,18732,15311],{},[59,18734,508],{},[55,18736,13611],{},[89,18738,18739],{"encoding":91},"B=64",[29,18741,18743,18761],{"className":18742,"ariaHidden":61},[96],[29,18744,18746,18749,18752,18755,18758],{"className":18745},[100],[29,18747],{"className":18748,"style":3718},[104],[29,18750,15311],{"className":18751,"style":17886},[109,113],[29,18753],{"className":18754,"style":367},[171],[29,18756,508],{"className":18757},[371],[29,18759],{"className":18760,"style":367},[171],[29,18762,18764,18767],{"className":18763},[100],[29,18765],{"className":18766,"style":3272},[104],[29,18768,13611],{"className":18769},[109],[14654,18771,18772],{},[29,18773,18775,18793],{"className":18774},[32],[29,18776,18778],{"className":18777},[36],[38,18779,18780],{"xmlns":40},[42,18781,18782,18790],{},[45,18783,18784,18786,18788],{},[51,18785,15311],{},[59,18787,508],{},[55,18789,13709],{},[89,18791,18792],{"encoding":91},"B=512",[29,18794,18796,18814],{"className":18795,"ariaHidden":61},[96],[29,18797,18799,18802,18805,18808,18811],{"className":18798},[100],[29,18800],{"className":18801,"style":3718},[104],[29,18803,15311],{"className":18804,"style":17886},[109,113],[29,18806],{"className":18807,"style":367},[171],[29,18809,508],{"className":18810},[371],[29,18812],{"className":18813,"style":367},[171],[29,18815,18817,18820],{"className":18816},[100],[29,18818],{"className":18819,"style":3272},[104],[29,18821,13709],{"className":18822},[109],[14654,18824,18825],{},[29,18826,18828,18846],{"className":18827},[32],[29,18829,18831],{"className":18830},[36],[38,18832,18833],{"xmlns":40},[42,18834,18835,18843],{},[45,18836,18837,18839,18841],{},[51,18838,15311],{},[59,18840,8389],{},[51,18842,17294],{"mathvariant":1002},[89,18844,18845],{"encoding":91},"B \\to \\infty",[29,18847,18849,18867],{"className":18848,"ariaHidden":61},[96],[29,18850,18852,18855,18858,18861,18864],{"className":18851},[100],[29,18853],{"className":18854,"style":3718},[104],[29,18856,15311],{"className":18857,"style":17886},[109,113],[29,18859],{"className":18860,"style":367},[171],[29,18862,8389],{"className":18863},[371],[29,18865],{"className":18866,"style":367},[171],[29,18868,18870,18873],{"className":18869},[100],[29,18871],{"className":18872,"style":537},[104],[29,18874,17294],{"className":18875},[109],[14663,18877,18878,18900,18918,18936],{},[14651,18879,18880,18882,18884,18887,18890,18895],{},[14668,18881,17788],{},[14668,18883,17791],{},[14668,18885,18886],{},"20.0",[14668,18888,18889],{},"118",[14668,18891,18892],{},[14,18893,18894],{},"303",[14668,18896,18897],{},[14,18898,18899],{},"390",[14651,18901,18902,18904,18906,18909,18912,18915],{},[14668,18903,17805],{},[14668,18905,17808],{},[14668,18907,18908],{},"78.5",[14668,18910,18911],{},"202",[14668,18913,18914],{},"251",[14668,18916,18917],{},"260",[14651,18919,18920,18922,18924,18927,18930,18933],{},[14668,18921,17821],{},[14668,18923,17824],{},[14668,18925,18926],{},"156",[14668,18928,18929],{},"230",[14668,18931,18932],{},"244",[14668,18934,18935],{},"246",[14651,18937,18938,18940,18942,18945,18948,18950],{},[14668,18939,17837],{},[14668,18941,17840],{},[14668,18943,18944],{},"226",[14668,18946,18947],{},"240",[14668,18949,17331],{},[14668,18951,17331],{},[10,18953,18954],{},"（加粗的是越过 295 的格子。）",[10,18956,18957],{},"于是结论要分区间说：",[15609,18959,18960,18966],{},[15612,18961,18962,18965],{},[14,18963,18964],{},"短 context（4k 这一档）","：batch 开得够大，decode 会翻到 compute bound。这时候那 4.25 倍的计算量是真的要还的。",[15612,18967,18968,18971],{},[14,18969,18970],{},"32k 及以上","：即使 batch 开到无穷，算术强度也够不到 295。decode 始终 memory bound，那 4.25 倍照样白加。",[10,18973,18974,18975,19003,19004,19090,19091,19124,19125,19281],{},"原因在渐近值里写得清楚。",[29,18976,18978,18991],{"className":18977},[32],[29,18979,18981],{"className":18980},[36],[38,18982,18983],{"xmlns":40},[42,18984,18985,18989],{},[45,18986,18987],{},[51,18988,3534],{},[89,18990,3534],{"encoding":91},[29,18992,18994],{"className":18993,"ariaHidden":61},[96],[29,18995,18997,19000],{"className":18996},[100],[29,18998],{"className":18999,"style":3718},[104],[29,19001,3534],{"className":19002,"style":3667},[109,113]," 一大，",[29,19005,19007,19029],{"className":19006},[32],[29,19008,19010],{"className":19009},[36],[38,19011,19012],{"xmlns":40},[42,19013,19014,19026],{},[45,19015,19016,19022,19024],{},[48,19017,19018,19020],{},[51,19019,6467],{},[76,19021,16004],{},[51,19023,3517],{},[51,19025,3534],{},[89,19027,19028],{"encoding":91},"c_{\\text{flop}} L N",[29,19030,19032],{"className":19031,"ariaHidden":61},[96],[29,19033,19035,19038,19084,19087],{"className":19034},[100],[29,19036],{"className":19037,"style":4236},[104],[29,19039,19041,19044],{"className":19040},[109],[29,19042,6467],{"className":19043},[109,113],[29,19045,19047],{"className":19046},[117],[29,19048,19050,19076],{"className":19049},[121,122],[29,19051,19053,19073],{"className":19052},[126],[29,19054,19056],{"className":19055,"style":253},[130],[29,19057,19058,19061],{"style":134},[29,19059],{"className":19060,"style":139},[138],[29,19062,19064],{"className":19063},[143,144,145,146],[29,19065,19067],{"className":19066},[109,146],[29,19068,19070],{"className":19069},[109,3686,146],[29,19071,16004],{"className":19072},[109,146],[29,19074,154],{"className":19075},[153],[29,19077,19079],{"className":19078},[126],[29,19080,19082],{"className":19081,"style":4281},[130],[29,19083],{},[29,19085,3517],{"className":19086},[109,113],[29,19088,3534],{"className":19089,"style":3667},[109,113]," 就压过 ",[29,19092,19094,19109],{"className":19093},[32],[29,19095,19097],{"className":19096},[36],[38,19098,19099],{"xmlns":40},[42,19100,19101,19107],{},[45,19102,19103,19105],{},[55,19104,69],{},[51,19106,16524],{},[89,19108,16847],{"encoding":91},[29,19110,19112],{"className":19111,"ariaHidden":61},[96],[29,19113,19115,19118,19121],{"className":19114},[100],[29,19116],{"className":19117,"style":3718},[104],[29,19119,69],{"className":19120},[109],[29,19122,16524],{"className":19123,"style":2350},[109,113],"，权重项被稀释掉，剩下的比值就是 ",[29,19126,19128,19158],{"className":19127},[32],[29,19129,19131],{"className":19130},[36],[38,19132,19133],{"xmlns":40},[42,19134,19135,19155],{},[45,19136,19137,19143,19145,19151,19153],{},[48,19138,19139,19141],{},[51,19140,6467],{},[76,19142,16004],{},[51,19144,3083],{"mathvariant":1002},[48,19146,19147,19149],{},[51,19148,6467],{},[76,19150,15716],{},[59,19152,508],{},[55,19154,17331],{},[89,19156,19157],{"encoding":91},"c_{\\text{flop}}\u002Fc_{\\text{kv}} = 242",[29,19159,19161,19272],{"className":19160,"ariaHidden":61},[96],[29,19162,19164,19168,19214,19217,19263,19266,19269],{"className":19163},[100],[29,19165],{"className":19166,"style":19167},[104],"height:1.0361em;vertical-align:-0.2861em;",[29,19169,19171,19174],{"className":19170},[109],[29,19172,6467],{"className":19173},[109,113],[29,19175,19177],{"className":19176},[117],[29,19178,19180,19206],{"className":19179},[121,122],[29,19181,19183,19203],{"className":19182},[126],[29,19184,19186],{"className":19185,"style":253},[130],[29,19187,19188,19191],{"style":134},[29,19189],{"className":19190,"style":139},[138],[29,19192,19194],{"className":19193},[143,144,145,146],[29,19195,19197],{"className":19196},[109,146],[29,19198,19200],{"className":19199},[109,3686,146],[29,19201,16004],{"className":19202},[109,146],[29,19204,154],{"className":19205},[153],[29,19207,19209],{"className":19208},[126],[29,19210,19212],{"className":19211,"style":4281},[130],[29,19213],{},[29,19215,3083],{"className":19216},[109],[29,19218,19220,19223],{"className":19219},[109],[29,19221,6467],{"className":19222},[109,113],[29,19224,19226],{"className":19225},[117],[29,19227,19229,19255],{"className":19228},[121,122],[29,19230,19232,19252],{"className":19231},[126],[29,19233,19235],{"className":19234,"style":253},[130],[29,19236,19237,19240],{"style":134},[29,19238],{"className":19239,"style":139},[138],[29,19241,19243],{"className":19242},[143,144,145,146],[29,19244,19246],{"className":19245},[109,146],[29,19247,19249],{"className":19248},[109,3686,146],[29,19250,15716],{"className":19251},[109,146],[29,19253,154],{"className":19254},[153],[29,19256,19258],{"className":19257},[126],[29,19259,19261],{"className":19260,"style":161},[130],[29,19262],{},[29,19264],{"className":19265,"style":367},[171],[29,19267,508],{"className":19268},[371],[29,19270],{"className":19271,"style":367},[171],[29,19273,19275,19278],{"className":19274},[100],[29,19276],{"className":19277,"style":3272},[104],[29,19279,17331],{"className":19280},[109],"，跟 batch 无关。",[10,19283,19284,19285],{},"所以 batch 只在短 context 上有效，而短 context 恰恰是 MLA 收益最小的区间。前面那张加速比表里，4k 只有 1.4×。两件事碰在一起，这个取舍就更清楚了：",[14,19286,19287],{},"MLA 在自己最有价值的长 context 区间永远是 memory bound，那笔计算量从头到尾都不用还。",[10,19289,19290,19291,413],{},"长 context 下 batch 也开不大，还有另一个原因：KV Cache 装不下。MLA 在 128k 时每条序列 9.06 GB，32 条就是 290 GB，早就超出单卡。而同样条件下的 MHA 是 515 GB，",[14,19292,19293],{},"一条序列都放不下",[10,19295,19296],{},"所以 MHA 在长 context 下跑不起来，谈不上快慢。",[10,19298,19299],{},"那\"划算\"的刻度在哪里？MLA 省的是字节，而字节由权重和 KV 两部分组成。KV 读取追平权重的位置：",[29,19301,19303],{"className":19302},[597],[29,19304,19306,19351],{"className":19305},[32],[29,19307,19309],{"className":19308},[36],[38,19310,19311],{"xmlns":40,"display":606},[42,19312,19313,19348],{},[45,19314,19315,19318,19321,19324,19326,19328,19330,19332,19335,19338,19340,19342,19345],{},[76,19316,19317],{},"MHA： ",[55,19319,19320],{},"3.93",[76,19322,19323],{}," MB",[59,19325,3514],{},[51,19327,3534],{},[59,19329,508],{},[55,19331,16340],{},[76,19333,19334],{}," GB",[59,19336,19337],{},"⇒",[51,19339,3534],{},[59,19341,3843],{},[55,19343,19344],{},"1.1",[76,19346,19347],{}," 万",[89,19349,19350],{"encoding":91},"\\text{MHA：}\\ 3.93\\ \\text{MB} \\times N = 42\\ \\text{GB} \\Rightarrow N \\approx 1.1\\ \\text{万}",[29,19352,19354,19394,19412,19440,19458],{"className":19353,"ariaHidden":61},[96],[29,19355,19357,19360,19369,19372,19375,19378,19385,19388,19391],{"className":19356},[100],[29,19358],{"className":19359,"style":3571},[104],[29,19361,19363,19366],{"className":19362},[109,3686],[29,19364,14670],{"className":19365},[109],[29,19367,7323],{"className":19368},[109,16407],[29,19370,5624],{"className":19371},[171],[29,19373,19320],{"className":19374},[109],[29,19376,5624],{"className":19377},[171],[29,19379,19381],{"className":19380},[109,3686],[29,19382,19384],{"className":19383},[109],"MB",[29,19386],{"className":19387,"style":3258},[171],[29,19389,3514],{"className":19390},[3262],[29,19392],{"className":19393,"style":3258},[171],[29,19395,19397,19400,19403,19406,19409],{"className":19396},[100],[29,19398],{"className":19399,"style":3718},[104],[29,19401,3534],{"className":19402,"style":3667},[109,113],[29,19404],{"className":19405,"style":367},[171],[29,19407,508],{"className":19408},[371],[29,19410],{"className":19411,"style":367},[171],[29,19413,19415,19418,19421,19424,19431,19434,19437],{"className":19414},[100],[29,19416],{"className":19417,"style":3718},[104],[29,19419,16340],{"className":19420},[109],[29,19422,5624],{"className":19423},[171],[29,19425,19427],{"className":19426},[109,3686],[29,19428,19430],{"className":19429},[109],"GB",[29,19432],{"className":19433,"style":367},[171],[29,19435,19337],{"className":19436},[371],[29,19438],{"className":19439,"style":367},[171],[29,19441,19443,19446,19449,19452,19455],{"className":19442},[100],[29,19444],{"className":19445,"style":3718},[104],[29,19447,3534],{"className":19448,"style":3667},[109,113],[29,19450],{"className":19451,"style":367},[171],[29,19453,3843],{"className":19454},[371],[29,19456],{"className":19457,"style":367},[171],[29,19459,19461,19464,19467,19470],{"className":19460},[100],[29,19462],{"className":19463,"style":3718},[104],[29,19465,19344],{"className":19466},[109],[29,19468,5624],{"className":19469},[171],[29,19471,19473],{"className":19472},[109,3686],[29,19474,19476],{"className":19475},[109,16407],"万",[29,19478,19480],{"className":19479},[597],[29,19481,19483,19525],{"className":19482},[32],[29,19484,19486],{"className":19485},[36],[38,19487,19488],{"xmlns":40,"display":606},[42,19489,19490,19522],{},[45,19491,19492,19495,19498,19501,19503,19505,19507,19509,19511,19513,19515,19517,19520],{},[76,19493,19494],{},"MLA： ",[55,19496,19497],{},"69.1",[76,19499,19500],{}," KB",[59,19502,3514],{},[51,19504,3534],{},[59,19506,508],{},[55,19508,16340],{},[76,19510,19334],{},[59,19512,19337],{},[51,19514,3534],{},[59,19516,3843],{},[55,19518,19519],{},"61",[76,19521,19347],{},[89,19523,19524],{"encoding":91},"\\text{MLA：}\\ 69.1\\ \\text{KB} \\times N = 42\\ \\text{GB} \\Rightarrow N \\approx 61\\ \\text{万}",[29,19526,19528,19568,19586,19613,19631],{"className":19527,"ariaHidden":61},[96],[29,19529,19531,19534,19543,19546,19549,19552,19559,19562,19565],{"className":19530},[100],[29,19532],{"className":19533,"style":3571},[104],[29,19535,19537,19540],{"className":19536},[109,3686],[29,19538,6444],{"className":19539},[109],[29,19541,7323],{"className":19542},[109,16407],[29,19544,5624],{"className":19545},[171],[29,19547,19497],{"className":19548},[109],[29,19550,5624],{"className":19551},[171],[29,19553,19555],{"className":19554},[109,3686],[29,19556,19558],{"className":19557},[109],"KB",[29,19560],{"className":19561,"style":3258},[171],[29,19563,3514],{"className":19564},[3262],[29,19566],{"className":19567,"style":3258},[171],[29,19569,19571,19574,19577,19580,19583],{"className":19570},[100],[29,19572],{"className":19573,"style":3718},[104],[29,19575,3534],{"className":19576,"style":3667},[109,113],[29,19578],{"className":19579,"style":367},[171],[29,19581,508],{"className":19582},[371],[29,19584],{"className":19585,"style":367},[171],[29,19587,19589,19592,19595,19598,19604,19607,19610],{"className":19588},[100],[29,19590],{"className":19591,"style":3718},[104],[29,19593,16340],{"className":19594},[109],[29,19596,5624],{"className":19597},[171],[29,19599,19601],{"className":19600},[109,3686],[29,19602,19430],{"className":19603},[109],[29,19605],{"className":19606,"style":367},[171],[29,19608,19337],{"className":19609},[371],[29,19611],{"className":19612,"style":367},[171],[29,19614,19616,19619,19622,19625,19628],{"className":19615},[100],[29,19617],{"className":19618,"style":3718},[104],[29,19620,3534],{"className":19621,"style":3667},[109,113],[29,19623],{"className":19624,"style":367},[171],[29,19626,3843],{"className":19627},[371],[29,19629],{"className":19630,"style":367},[171],[29,19632,19634,19637,19640,19643],{"className":19633},[100],[29,19635],{"className":19636,"style":3718},[104],[29,19638,19519],{"className":19639},[109],[29,19641,5624],{"className":19642},[171],[29,19644,19646],{"className":19645},[109,3686],[29,19647,19476],{"className":19648},[109,16407],[10,19650,19651],{},"既然两边都是 memory bound，加速比就等于字节数之比：",[14645,19653,19654,19691],{},[14648,19655,19656],{},[14651,19657,19658,19688],{},[14654,19659,19660],{},[29,19661,19663,19676],{"className":19662},[32],[29,19664,19666],{"className":19665},[36],[38,19667,19668],{"xmlns":40},[42,19669,19670,19674],{},[45,19671,19672],{},[51,19673,3534],{},[89,19675,3534],{"encoding":91},[29,19677,19679],{"className":19678,"ariaHidden":61},[96],[29,19680,19682,19685],{"className":19681},[100],[29,19683],{"className":19684,"style":3718},[104],[29,19686,3534],{"className":19687,"style":3667},[109,113],[14654,19689,19690],{},"MLA 相对 MHA 的每 token 加速比",[14663,19692,19693,19700,19707,19714],{},[14651,19694,19695,19697],{},[14668,19696,17788],{},[14668,19698,19699],{},"1.4×",[14651,19701,19702,19704],{},[14668,19703,17805],{},[14668,19705,19706],{},"3.9×",[14651,19708,19709,19711],{},[14668,19710,17821],{},[14668,19712,19713],{},"10.9×",[14651,19715,19716,19718],{},[14668,19717,17837],{},[14668,19719,19720],{},"36.4×",[10,19722,19723,19724],{},"所以答案是一条具体的曲线：",[14,19725,19726],{},"4k 时几乎白干（1.4×），32k 开始有意义（3.9×），128k 是 11 倍，1M 是 36 倍。",[10,19728,19729],{},"这条曲线顺带解释了 MLA 的时间线。它 2024 年出来的时候，主流 context 还在 4k 到 32k 之间，那一段的收益并不惊人；让它值回票价的是后面 128k、1M 的军备竞赛。",[10,19731,19732,19733,19847,19848,20011],{},"另外还有一件事能说明这个设计有多自觉。一般 LLM 架构满足 ",[29,19734,19736,19762],{"className":19735},[32],[29,19737,19739],{"className":19738},[36],[38,19740,19741],{"xmlns":40},[42,19742,19743,19759],{},[45,19744,19745,19747,19749,19755,19757],{},[51,19746,428],{},[59,19748,3514],{},[48,19750,19751,19753],{},[51,19752,309],{},[51,19754,1045],{},[59,19756,508],{},[51,19758,309],{},[89,19760,19761],{"encoding":91},"h \\times d_k = d",[29,19763,19765,19783,19838],{"className":19764,"ariaHidden":61},[96],[29,19766,19768,19771,19774,19777,19780],{"className":19767},[100],[29,19769],{"className":19770,"style":3590},[104],[29,19772,428],{"className":19773},[109,113],[29,19775],{"className":19776,"style":3258},[171],[29,19778,3514],{"className":19779},[3262],[29,19781],{"className":19782,"style":3258},[171],[29,19784,19786,19789,19829,19832,19835],{"className":19785},[100],[29,19787],{"className":19788,"style":3098},[104],[29,19790,19792,19795],{"className":19791},[109],[29,19793,309],{"className":19794},[109,113],[29,19796,19798],{"className":19797},[117],[29,19799,19801,19821],{"className":19800},[121,122],[29,19802,19804,19818],{"className":19803},[126],[29,19805,19807],{"className":19806,"style":253},[130],[29,19808,19809,19812],{"style":134},[29,19810],{"className":19811,"style":139},[138],[29,19813,19815],{"className":19814},[143,144,145,146],[29,19816,1045],{"className":19817,"style":1387},[109,113,146],[29,19819,154],{"className":19820},[153],[29,19822,19824],{"className":19823},[126],[29,19825,19827],{"className":19826,"style":161},[130],[29,19828],{},[29,19830],{"className":19831,"style":367},[171],[29,19833,508],{"className":19834},[371],[29,19836],{"className":19837,"style":367},[171],[29,19839,19841,19844],{"className":19840},[100],[29,19842],{"className":19843,"style":440},[104],[29,19845,309],{"className":19846},[109,113],"，但 DeepSeek-V2 是 ",[29,19849,19851,19890],{"className":19850},[32],[29,19852,19854],{"className":19853},[36],[38,19855,19856],{"xmlns":40},[42,19857,19858,19887],{},[45,19859,19860,19866,19868,19870,19872,19874,19876,19879,19881,19883,19885],{},[48,19861,19862,19864],{},[51,19863,309],{},[51,19865,1045],{},[59,19867,508],{},[55,19869,3840],{},[59,19871,62],{"separator":61},[51,19873,309],{},[59,19875,508],{},[55,19877,19878],{},"5120",[59,19880,62],{"separator":61},[51,19882,428],{},[59,19884,508],{},[55,19886,3840],{},[89,19888,19889],{"encoding":91},"d_k = 128, d = 5120, h = 128",[29,19891,19893,19948,19975,20002],{"className":19892,"ariaHidden":61},[96],[29,19894,19896,19899,19939,19942,19945],{"className":19895},[100],[29,19897],{"className":19898,"style":3098},[104],[29,19900,19902,19905],{"className":19901},[109],[29,19903,309],{"className":19904},[109,113],[29,19906,19908],{"className":19907},[117],[29,19909,19911,19931],{"className":19910},[121,122],[29,19912,19914,19928],{"className":19913},[126],[29,19915,19917],{"className":19916,"style":253},[130],[29,19918,19919,19922],{"style":134},[29,19920],{"className":19921,"style":139},[138],[29,19923,19925],{"className":19924},[143,144,145,146],[29,19926,1045],{"className":19927,"style":1387},[109,113,146],[29,19929,154],{"className":19930},[153],[29,19932,19934],{"className":19933},[126],[29,19935,19937],{"className":19936,"style":161},[130],[29,19938],{},[29,19940],{"className":19941,"style":367},[171],[29,19943,508],{"className":19944},[371],[29,19946],{"className":19947,"style":367},[171],[29,19949,19951,19954,19957,19960,19963,19966,19969,19972],{"className":19950},[100],[29,19952],{"className":19953,"style":556},[104],[29,19955,3840],{"className":19956},[109],[29,19958,62],{"className":19959},[167],[29,19961],{"className":19962,"style":172},[171],[29,19964,309],{"className":19965},[109,113],[29,19967],{"className":19968,"style":367},[171],[29,19970,508],{"className":19971},[371],[29,19973],{"className":19974,"style":367},[171],[29,19976,19978,19981,19984,19987,19990,19993,19996,19999],{"className":19977},[100],[29,19979],{"className":19980,"style":556},[104],[29,19982,19878],{"className":19983},[109],[29,19985,62],{"className":19986},[167],[29,19988],{"className":19989,"style":172},[171],[29,19991,428],{"className":19992},[109,113],[29,19994],{"className":19995,"style":367},[171],[29,19997,508],{"className":19998},[371],[29,20000],{"className":20001,"style":367},[171],[29,20003,20005,20008],{"className":20004},[100],[29,20006],{"className":20007,"style":3272},[104],[29,20009,3840],{"className":20010},[109],"，head 数是一般设置的 3 倍。",[10,20013,20014,20015,20043,20044,20072,20073,413],{},"因为 MLA 的 KV Cache 大小跟 ",[29,20016,20018,20031],{"className":20017},[32],[29,20019,20021],{"className":20020},[36],[38,20022,20023],{"xmlns":40},[42,20024,20025,20029],{},[45,20026,20027],{},[51,20028,428],{},[89,20030,428],{"encoding":91},[29,20032,20034],{"className":20033,"ariaHidden":61},[96],[29,20035,20037,20040],{"className":20036},[100],[29,20038],{"className":20039,"style":440},[104],[29,20041,428],{"className":20042},[109,113]," 无关。增大 ",[29,20045,20047,20060],{"className":20046},[32],[29,20048,20050],{"className":20049},[36],[38,20051,20052],{"xmlns":40},[42,20053,20054,20058],{},[45,20055,20056],{},[51,20057,428],{},[89,20059,428],{"encoding":91},[29,20061,20063],{"className":20062,"ariaHidden":61},[96],[29,20064,20066,20069],{"className":20065},[100],[29,20067],{"className":20068,"style":440},[104],[29,20070,428],{"className":20071},[109,113]," 只增加计算量和提升模型能力，不增加 KV Cache，所以不会带来速度瓶颈。MLA 完成的思想升级：",[14,20074,20075],{},"从离散的 head sharing，到连续的 representation compression",[21,20077,20079],{"id":20078},"csa-与-hca","CSA 与 HCA",[20081,20082,20084],"h3",{"id":20083},"压-sequence-轴","压 Sequence 轴",[10,20086,20087,20088,20288,20289,20317,20318,20369],{},"MLA 压小的是每个 token 存的东西。但 ",[29,20089,20091,20131],{"className":20090},[32],[29,20092,20094],{"className":20093},[36],[38,20095,20096],{"xmlns":40},[42,20097,20098,20128],{},[45,20099,20100,20106,20108,20114,20116,20118,20120,20122],{},[48,20101,20102,20104],{},[51,20103,6467],{},[55,20105,57],{},[59,20107,62],{"separator":61},[48,20109,20110,20112],{},[51,20111,6467],{},[55,20113,69],{},[59,20115,62],{"separator":61},[59,20117,74],{},[76,20119,78],{},[59,20121,62],{"separator":61},[48,20123,20124,20126],{},[51,20125,6467],{},[51,20127,3534],{},[89,20129,20130],{"encoding":91},"c_1, c_2, \\cdots, c_N",[29,20132,20134],{"className":20133,"ariaHidden":61},[96],[29,20135,20137,20140,20180,20183,20186,20226,20229,20232,20235,20238,20241,20244,20247],{"className":20136},[100],[29,20138],{"className":20139,"style":105},[104],[29,20141,20143,20146],{"className":20142},[109],[29,20144,6467],{"className":20145},[109,113],[29,20147,20149],{"className":20148},[117],[29,20150,20152,20172],{"className":20151},[121,122],[29,20153,20155,20169],{"className":20154},[126],[29,20156,20158],{"className":20157,"style":131},[130],[29,20159,20160,20163],{"style":134},[29,20161],{"className":20162,"style":139},[138],[29,20164,20166],{"className":20165},[143,144,145,146],[29,20167,57],{"className":20168},[109,146],[29,20170,154],{"className":20171},[153],[29,20173,20175],{"className":20174},[126],[29,20176,20178],{"className":20177,"style":161},[130],[29,20179],{},[29,20181,62],{"className":20182},[167],[29,20184],{"className":20185,"style":172},[171],[29,20187,20189,20192],{"className":20188},[109],[29,20190,6467],{"className":20191},[109,113],[29,20193,20195],{"className":20194},[117],[29,20196,20198,20218],{"className":20197},[121,122],[29,20199,20201,20215],{"className":20200},[126],[29,20202,20204],{"className":20203,"style":131},[130],[29,20205,20206,20209],{"style":134},[29,20207],{"className":20208,"style":139},[138],[29,20210,20212],{"className":20211},[143,144,145,146],[29,20213,69],{"className":20214},[109,146],[29,20216,154],{"className":20217},[153],[29,20219,20221],{"className":20220},[126],[29,20222,20224],{"className":20223,"style":161},[130],[29,20225],{},[29,20227,62],{"className":20228},[167],[29,20230],{"className":20231,"style":172},[171],[29,20233,74],{"className":20234},[222],[29,20236],{"className":20237,"style":172},[171],[29,20239],{"className":20240,"style":172},[171],[29,20242,62],{"className":20243},[167],[29,20245],{"className":20246,"style":172},[171],[29,20248,20250,20253],{"className":20249},[109],[29,20251,6467],{"className":20252},[109,113],[29,20254,20256],{"className":20255},[117],[29,20257,20259,20280],{"className":20258},[121,122],[29,20260,20262,20277],{"className":20261},[126],[29,20263,20266],{"className":20264,"style":20265},[130],"height:0.3283em;",[29,20267,20268,20271],{"style":134},[29,20269],{"className":20270,"style":139},[138],[29,20272,20274],{"className":20273},[143,144,145,146],[29,20275,3534],{"className":20276,"style":3667},[109,113,146],[29,20278,154],{"className":20279},[153],[29,20281,20283],{"className":20282},[126],[29,20284,20286],{"className":20285,"style":161},[130],[29,20287],{}," 仍然有 ",[29,20290,20292,20305],{"className":20291},[32],[29,20293,20295],{"className":20294},[36],[38,20296,20297],{"xmlns":40},[42,20298,20299,20303],{},[45,20300,20301],{},[51,20302,3534],{},[89,20304,3534],{"encoding":91},[29,20306,20308],{"className":20307,"ariaHidden":61},[96],[29,20309,20311,20314],{"className":20310},[100],[29,20312],{"className":20313,"style":3718},[104],[29,20315,3534],{"className":20316,"style":3667},[109,113]," 个。如果每个 query 还是要看全部历史，那 ",[29,20319,20321,20339],{"className":20320},[32],[29,20322,20324],{"className":20323},[36],[38,20325,20326],{"xmlns":40},[42,20327,20328,20336],{},[45,20329,20330,20332,20334],{},[51,20331,3534],{},[59,20333,3514],{},[51,20335,3534],{},[89,20337,20338],{"encoding":91},"N \\times N",[29,20340,20342,20360],{"className":20341,"ariaHidden":61},[96],[29,20343,20345,20348,20351,20354,20357],{"className":20344},[100],[29,20346],{"className":20347,"style":3571},[104],[29,20349,3534],{"className":20350,"style":3667},[109,113],[29,20352],{"className":20353,"style":3258},[171],[29,20355,3514],{"className":20356},[3262],[29,20358],{"className":20359,"style":3258},[171],[29,20361,20363,20366],{"className":20362},[100],[29,20364],{"className":20365,"style":3718},[104],[29,20367,3534],{"className":20368,"style":3667},[109,113]," 还在。",[10,20371,20372],{},[14,20373,20374],{},"MLA 解决的是 KV 的宽度，没解决历史的长度。",[10,20376,20377],{},"把 KV Cache 想成一个二维矩阵：",[29,20379,20381],{"className":20380},[597],[29,20382,20384,20404],{"className":20383},[32],[29,20385,20387],{"className":20386},[36],[38,20388,20389],{"xmlns":40,"display":606},[42,20390,20391,20401],{},[45,20392,20393,20396,20398],{},[76,20394,20395],{},"Sequence",[59,20397,3514],{},[76,20399,20400],{},"Feature",[89,20402,20403],{"encoding":91},"\\text{Sequence} \\times \\text{Feature}",[29,20405,20407,20428],{"className":20406,"ariaHidden":61},[96],[29,20408,20410,20413,20419,20422,20425],{"className":20409},[100],[29,20411],{"className":20412,"style":7549},[104],[29,20414,20416],{"className":20415},[109,3686],[29,20417,20395],{"className":20418},[109],[29,20420],{"className":20421,"style":3258},[171],[29,20423,3514],{"className":20424},[3262],[29,20426],{"className":20427,"style":3258},[171],[29,20429,20431,20434],{"className":20430},[100],[29,20432],{"className":20433,"style":3718},[104],[29,20435,20437],{"className":20436},[109,3686],[29,20438,20400],{"className":20439},[109],[10,20441,20442],{},"MQA、GQA、MLA 压的都是 Feature 轴。V4 开始压 Sequence 轴。V4 改成了三类 attention：",[15609,20444,20445,20451,20457],{},[15612,20446,20447,20450],{},[14,20448,20449],{},"CSA","：先按大约 4:1 沿序列压缩，再用 Lightning Indexer 给每个 query 挑少量 top-k compressed blocks",[15612,20452,20453,20456],{},[14,20454,20455],{},"HCA","：用更激进的序列压缩（约 128 倍），让 query 对压缩后的全局历史做 attention",[15612,20458,20459,20462],{},[14,20460,20461],{},"Sliding Window","：保留最近的精细上下文",[10,20464,20465],{},"这两个压缩率不是估的。Hugging Face 对 V4 的实现文档给出了默认值：CSA 是 4，HCA 是 128。为什么非要留一个 sliding window？因为压缩是有损的。CSA 压到 1\u002F4，HCA 压到 1\u002F128，最近的几个 token 一旦被压进 block，细节就再也找不回来了。而语言模型对紧接着的上下文又格外敏感。代词指谁、缩进几个空格、刚才那个变量叫什么名字、上一个 tool call 返回了什么。",[10,20467,20468],{},"这些信息全都发生在最后几百个 token 里。",[10,20470,20471,20472,20522,20523,20551,20552,20580,20581,20609],{},"所以最近的一段必须原样保留。三类 attention 是三段分工，不能互相替换：最近的原样看，中间挑着看，最远的粗着看。这个分工其实很符合直觉。人读一篇长文档也是这样：最后一段逐字读，中间扫小标题，开头的细节早忘了，只记得大意。于是 attention 不再是简单的 ",[29,20473,20475,20492],{"className":20474},[32],[29,20476,20478],{"className":20477},[36],[38,20479,20480],{"xmlns":40},[42,20481,20482,20490],{},[45,20483,20484,20486,20488],{},[51,20485,3534],{},[59,20487,3514],{},[51,20489,3534],{},[89,20491,20338],{"encoding":91},[29,20493,20495,20513],{"className":20494,"ariaHidden":61},[96],[29,20496,20498,20501,20504,20507,20510],{"className":20497},[100],[29,20499],{"className":20500,"style":3571},[104],[29,20502,3534],{"className":20503,"style":3667},[109,113],[29,20505],{"className":20506,"style":3258},[171],[29,20508,3514],{"className":20509},[3262],[29,20511],{"className":20512,"style":3258},[171],[29,20514,20516,20519],{"className":20515},[100],[29,20517],{"className":20518,"style":3718},[104],[29,20520,3534],{"className":20521,"style":3667},[109,113],"，而是三种东西的叠加（记局部窗口大小为 ",[29,20524,20526,20539],{"className":20525},[32],[29,20527,20529],{"className":20528},[36],[38,20530,20531],{"xmlns":40},[42,20532,20533,20537],{},[45,20534,20535],{},[51,20536,2087],{},[89,20538,2087],{"encoding":91},[29,20540,20542],{"className":20541,"ariaHidden":61},[96],[29,20543,20545,20548],{"className":20544},[100],[29,20546],{"className":20547,"style":3718},[104],[29,20549,2087],{"className":20550,"style":2350},[109,113],"，每个 query 选中的 block 数为 ",[29,20553,20555,20568],{"className":20554},[32],[29,20556,20558],{"className":20557},[36],[38,20559,20560],{"xmlns":40},[42,20561,20562,20566],{},[45,20563,20564],{},[51,20565,15311],{},[89,20567,15311],{"encoding":91},[29,20569,20571],{"className":20570,"ariaHidden":61},[96],[29,20572,20574,20577],{"className":20573},[100],[29,20575],{"className":20576,"style":3718},[104],[29,20578,15311],{"className":20579,"style":17886},[109,113],"，压缩率为 ",[29,20582,20584,20597],{"className":20583},[32],[29,20585,20587],{"className":20586},[36],[38,20588,20589],{"xmlns":40},[42,20590,20591,20595],{},[45,20592,20593],{},[51,20594,13458],{},[89,20596,13458],{"encoding":91},[29,20598,20600],{"className":20599,"ariaHidden":61},[96],[29,20601,20603,20606],{"className":20602},[100],[29,20604],{"className":20605,"style":537},[104],[29,20607,13458],{"className":20608,"style":4465},[109,113],"）：",[29,20611,20613],{"className":20612},[597],[29,20614,20616,20639],{"className":20615},[32],[29,20617,20619],{"className":20618},[36],[38,20620,20621],{"xmlns":40,"display":606},[42,20622,20623,20636],{},[45,20624,20625,20627,20629,20631,20633],{},[51,20626,3534],{},[59,20628,3514],{},[51,20630,2087],{},[171,20632],{"width":2114},[76,20634,20635],{},"（局部窗口）",[89,20637,20638],{"encoding":91},"N \\times W \\quad \\text{（局部窗口）}",[29,20640,20642,20660],{"className":20641,"ariaHidden":61},[96],[29,20643,20645,20648,20651,20654,20657],{"className":20644},[100],[29,20646],{"className":20647,"style":3571},[104],[29,20649,3534],{"className":20650,"style":3667},[109,113],[29,20652],{"className":20653,"style":3258},[171],[29,20655,3514],{"className":20656},[3262],[29,20658],{"className":20659,"style":3258},[171],[29,20661,20663,20666,20669,20672],{"className":20662},[100],[29,20664],{"className":20665,"style":3718},[104],[29,20667,2087],{"className":20668,"style":2350},[109,113],[29,20670],{"className":20671,"style":2506},[171],[29,20673,20675],{"className":20674},[109,3686],[29,20676,20635],{"className":20677},[109,16407],[29,20679,20681],{"className":20680},[597],[29,20682,20684,20707],{"className":20683},[32],[29,20685,20687],{"className":20686},[36],[38,20688,20689],{"xmlns":40,"display":606},[42,20690,20691,20704],{},[45,20692,20693,20695,20697,20699,20701],{},[51,20694,3534],{},[59,20696,3514],{},[51,20698,15311],{},[171,20700],{"width":2114},[76,20702,20703],{},"（稀疏选中的 compressed blocks）",[89,20705,20706],{"encoding":91},"N \\times B \\quad \\text{（稀疏选中的 compressed blocks）}",[29,20708,20710,20728],{"className":20709,"ariaHidden":61},[96],[29,20711,20713,20716,20719,20722,20725],{"className":20712},[100],[29,20714],{"className":20715,"style":3571},[104],[29,20717,3534],{"className":20718,"style":3667},[109,113],[29,20720],{"className":20721,"style":3258},[171],[29,20723,3514],{"className":20724},[3262],[29,20726],{"className":20727,"style":3258},[171],[29,20729,20731,20734,20737,20740],{"className":20730},[100],[29,20732],{"className":20733,"style":556},[104],[29,20735,15311],{"className":20736,"style":17886},[109,113],[29,20738],{"className":20739,"style":2506},[171],[29,20741,20743,20747,20751],{"className":20742},[109,3686],[29,20744,20746],{"className":20745},[109,16407],"（稀疏选中的",[29,20748,20750],{"className":20749},[109]," compressed blocks",[29,20752,20754],{"className":20753},[109,16407],"）",[29,20756,20758],{"className":20757},[597],[29,20759,20761,20788],{"className":20760},[32],[29,20762,20764],{"className":20763},[36],[38,20765,20766],{"xmlns":40,"display":606},[42,20767,20768,20785],{},[45,20769,20770,20772,20774,20780,20782],{},[51,20771,3534],{},[59,20773,3514],{},[1088,20775,20776,20778],{},[51,20777,3534],{},[51,20779,13458],{},[171,20781],{"width":2114},[76,20783,20784],{},"（高度压缩的 global branch）",[89,20786,20787],{"encoding":91},"N \\times \\frac{N}{r} \\quad \\text{（高度压缩的 global branch）}",[29,20789,20791,20809],{"className":20790,"ariaHidden":61},[96],[29,20792,20794,20797,20800,20803,20806],{"className":20793},[100],[29,20795],{"className":20796,"style":3571},[104],[29,20798,3534],{"className":20799,"style":3667},[109,113],[29,20801],{"className":20802,"style":3258},[171],[29,20804,3514],{"className":20805},[3262],[29,20807],{"className":20808,"style":3258},[171],[29,20810,20812,20816,20878,20881],{"className":20811},[100],[29,20813],{"className":20814,"style":20815},[104],"height:2.0463em;vertical-align:-0.686em;",[29,20817,20819,20822,20875],{"className":20818},[109],[29,20820],{"className":20821},[483,1549],[29,20823,20825],{"className":20824},[1088],[29,20826,20828,20867],{"className":20827},[121,122],[29,20829,20831,20864],{"className":20830},[126],[29,20832,20834,20845,20853],{"className":20833,"style":15511},[130],[29,20835,20836,20839],{"style":5413},[29,20837],{"className":20838,"style":5417},[138],[29,20840,20842],{"className":20841},[109],[29,20843,13458],{"className":20844,"style":4465},[109,113],[29,20846,20847,20850],{"style":5426},[29,20848],{"className":20849,"style":5417},[138],[29,20851],{"className":20852,"style":1776},[1775],[29,20854,20855,20858],{"style":5435},[29,20856],{"className":20857,"style":5417},[138],[29,20859,20861],{"className":20860},[109],[29,20862,3534],{"className":20863,"style":3667},[109,113],[29,20865,154],{"className":20866},[153],[29,20868,20870],{"className":20869},[126],[29,20871,20873],{"className":20872,"style":5457},[130],[29,20874],{},[29,20876],{"className":20877},[490,1549],[29,20879],{"className":20880,"style":2506},[171],[29,20882,20884,20888,20892],{"className":20883},[109,3686],[29,20885,20887],{"className":20886},[109,16407],"（高度压缩的",[29,20889,20891],{"className":20890},[109]," global branch",[29,20893,20754],{"className":20894},[109,16407],[20081,20896,20898],{"id":20897},"复杂度没有变成-on","复杂度没有变成 O(N)",[10,20900,20901,20902,20930,20931,21000,21001,413],{},"如果 HCA 那一路的 ",[29,20903,20905,20918],{"className":20904},[32],[29,20906,20908],{"className":20907},[36],[38,20909,20910],{"xmlns":40},[42,20911,20912,20916],{},[45,20913,20914],{},[51,20915,13458],{},[89,20917,13458],{"encoding":91},[29,20919,20921],{"className":20920,"ariaHidden":61},[96],[29,20922,20924,20927],{"className":20923},[100],[29,20925],{"className":20926,"style":537},[104],[29,20928,13458],{"className":20929,"style":4465},[109,113]," 是个固定常数，那么它的理论渐进复杂度仍然带有 ",[29,20932,20934,20956],{"className":20933},[32],[29,20935,20937],{"className":20936},[36],[38,20938,20939],{"xmlns":40},[42,20940,20941,20953],{},[45,20942,20943,20949,20951],{},[301,20944,20945,20947],{},[51,20946,3534],{},[55,20948,69],{},[51,20950,3083],{"mathvariant":1002},[51,20952,13458],{},[89,20954,20955],{"encoding":91},"N^2\u002Fr",[29,20957,20959],{"className":20958,"ariaHidden":61},[96],[29,20960,20962,20965,20994,20997],{"className":20961},[100],[29,20963],{"className":20964,"style":4461},[104],[29,20966,20968,20971],{"className":20967},[109],[29,20969,3534],{"className":20970,"style":3667},[109,113],[29,20972,20974],{"className":20973},[117],[29,20975,20977],{"className":20976},[121],[29,20978,20980],{"className":20979},[126],[29,20981,20983],{"className":20982,"style":4490},[130],[29,20984,20985,20988],{"style":403},[29,20986],{"className":20987,"style":139},[138],[29,20989,20991],{"className":20990},[143,144,145,146],[29,20992,69],{"className":20993},[109,146],[29,20995,3083],{"className":20996},[109],[29,20998,13458],{"className":20999,"style":4465},[109,113],"，没有数学意义上变成 ",[29,21002,21004,21024],{"className":21003},[32],[29,21005,21007],{"className":21006},[36],[38,21008,21009],{"xmlns":40},[42,21010,21011,21021],{},[45,21012,21013,21015,21017,21019],{},[51,21014,4436],{},[59,21016,460],{"stretchy":459},[51,21018,3534],{},[59,21020,466],{"stretchy":459},[89,21022,21023],{"encoding":91},"O(N)",[29,21025,21027],{"className":21026,"ariaHidden":61},[96],[29,21028,21030,21033,21036,21039,21042],{"className":21029},[100],[29,21031],{"className":21032,"style":479},[104],[29,21034,4436],{"className":21035,"style":4465},[109,113],[29,21037,460],{"className":21038},[483],[29,21040,3534],{"className":21041,"style":3667},[109,113],[29,21043,466],{"className":21044},[490],[10,21046,21047],{},"但工程量级已经差很多。而 CSA 的 sparse branch 更接近：",[29,21049,21051],{"className":21050},[597],[29,21052,21054,21087],{"className":21053},[32],[29,21055,21057],{"className":21056},[36],[38,21058,21059],{"xmlns":40,"display":606},[42,21060,21061,21084],{},[45,21062,21063,21065,21067,21069,21071,21073,21075,21077,21079,21082],{},[51,21064,4436],{},[59,21066,460],{"stretchy":459},[51,21068,3534],{},[51,21070,15311],{},[59,21072,466],{"stretchy":459},[59,21074,62],{"separator":61},[171,21076],{"width":2114},[51,21078,15311],{},[59,21080,21081],{},"≪",[51,21083,3534],{},[89,21085,21086],{"encoding":91},"O(NB), \\quad B \\ll N",[29,21088,21090,21132],{"className":21089,"ariaHidden":61},[96],[29,21091,21093,21096,21099,21102,21105,21108,21111,21114,21117,21120,21123,21126,21129],{"className":21092},[100],[29,21094],{"className":21095,"style":479},[104],[29,21097,4436],{"className":21098,"style":4465},[109,113],[29,21100,460],{"className":21101},[483],[29,21103,3534],{"className":21104,"style":3667},[109,113],[29,21106,15311],{"className":21107,"style":17886},[109,113],[29,21109,466],{"className":21110},[490],[29,21112,62],{"className":21113},[167],[29,21115],{"className":21116,"style":2506},[171],[29,21118],{"className":21119,"style":172},[171],[29,21121,15311],{"className":21122,"style":17886},[109,113],[29,21124],{"className":21125,"style":367},[171],[29,21127,21081],{"className":21128},[371],[29,21130],{"className":21131,"style":367},[171],[29,21133,21135,21138],{"className":21134},[100],[29,21136],{"className":21137,"style":3718},[104],[29,21139,3534],{"className":21140,"style":3667},[109,113],[10,21142,21143],{},"这是 V4 攻击 quadratic attention 的部分。所以\"V4 把 attention 变成线性了\"这种说法，把两件事混成了一件。严格结论和工程结论要分开讲。",[21,21145,21147],{"id":21146},"linear-attention-与-ssm","Linear Attention 与 SSM",[10,21149,21150,21151,21154],{},"前面所有方案都建立在同一个前提上：KV 要保存下来。那换个问题。",[14,21152,21153],{},"如果 KV 这么贵，为什么一定要保存它？"," RNN 的答案是最极端的：",[29,21156,21158],{"className":21157},[597],[29,21159,21161,21226],{"className":21160},[32],[29,21162,21164],{"className":21163},[36],[38,21165,21166],{"xmlns":40,"display":606},[42,21167,21168,21223],{},[45,21169,21170,21176,21178,21181,21183,21185,21191,21197,21199,21205,21217,21219,21221],{},[48,21171,21172,21174],{},[51,21173,428],{},[51,21175,618],{},[59,21177,508],{},[51,21179,21180],{},"tanh",[59,21182,1111],{},[59,21184,460],{"stretchy":459},[48,21186,21187,21189],{},[51,21188,2087],{},[51,21190,53],{},[48,21192,21193,21195],{},[51,21194,53],{},[51,21196,618],{},[59,21198,3236],{},[48,21200,21201,21203],{},[51,21202,2087],{},[51,21204,428],{},[48,21206,21207,21209],{},[51,21208,428],{},[45,21210,21211,21213,21215],{},[51,21212,618],{},[59,21214,12362],{},[55,21216,57],{},[59,21218,3236],{},[51,21220,14576],{},[59,21222,466],{"stretchy":459},[89,21224,21225],{"encoding":91},"h_t = \\tanh(W_x x_t + W_h h_{t-1} + b)",[29,21227,21229,21284,21385,21490],{"className":21228,"ariaHidden":61},[96],[29,21230,21232,21235,21275,21278,21281],{"className":21231},[100],[29,21233],{"className":21234,"style":3098},[104],[29,21236,21238,21241],{"className":21237},[109],[29,21239,428],{"className":21240},[109,113],[29,21242,21244],{"className":21243},[117],[29,21245,21247,21267],{"className":21246},[121,122],[29,21248,21250,21264],{"className":21249},[126],[29,21251,21253],{"className":21252,"style":711},[130],[29,21254,21255,21258],{"style":134},[29,21256],{"className":21257,"style":139},[138],[29,21259,21261],{"className":21260},[143,144,145,146],[29,21262,618],{"className":21263},[109,113,146],[29,21265,154],{"className":21266},[153],[29,21268,21270],{"className":21269},[126],[29,21271,21273],{"className":21272,"style":161},[130],[29,21274],{},[29,21276],{"className":21277,"style":367},[171],[29,21279,508],{"className":21280},[371],[29,21282],{"className":21283,"style":367},[171],[29,21285,21287,21290,21293,21296,21336,21376,21379,21382],{"className":21286},[100],[29,21288],{"className":21289,"style":479},[104],[29,21291,21180],{"className":21292},[1576],[29,21294,460],{"className":21295},[483],[29,21297,21299,21302],{"className":21298},[109],[29,21300,2087],{"className":21301,"style":2350},[109,113],[29,21303,21305],{"className":21304},[117],[29,21306,21308,21328],{"className":21307},[121,122],[29,21309,21311,21325],{"className":21310},[126],[29,21312,21314],{"className":21313,"style":3172},[130],[29,21315,21316,21319],{"style":7302},[29,21317],{"className":21318,"style":139},[138],[29,21320,21322],{"className":21321},[143,144,145,146],[29,21323,53],{"className":21324},[109,113,146],[29,21326,154],{"className":21327},[153],[29,21329,21331],{"className":21330},[126],[29,21332,21334],{"className":21333,"style":161},[130],[29,21335],{},[29,21337,21339,21342],{"className":21338},[109],[29,21340,53],{"className":21341},[109,113],[29,21343,21345],{"className":21344},[117],[29,21346,21348,21368],{"className":21347},[121,122],[29,21349,21351,21365],{"className":21350},[126],[29,21352,21354],{"className":21353,"style":711},[130],[29,21355,21356,21359],{"style":134},[29,21357],{"className":21358,"style":139},[138],[29,21360,21362],{"className":21361},[143,144,145,146],[29,21363,618],{"className":21364},[109,113,146],[29,21366,154],{"className":21367},[153],[29,21369,21371],{"className":21370},[126],[29,21372,21374],{"className":21373,"style":161},[130],[29,21375],{},[29,21377],{"className":21378,"style":3258},[171],[29,21380,3236],{"className":21381},[3262],[29,21383],{"className":21384,"style":3258},[171],[29,21386,21388,21392,21432,21481,21484,21487],{"className":21387},[100],[29,21389],{"className":21390,"style":21391},[104],"height:0.9028em;vertical-align:-0.2083em;",[29,21393,21395,21398],{"className":21394},[109],[29,21396,2087],{"className":21397,"style":2350},[109,113],[29,21399,21401],{"className":21400},[117],[29,21402,21404,21424],{"className":21403},[121,122],[29,21405,21407,21421],{"className":21406},[126],[29,21408,21410],{"className":21409,"style":253},[130],[29,21411,21412,21415],{"style":7302},[29,21413],{"className":21414,"style":139},[138],[29,21416,21418],{"className":21417},[143,144,145,146],[29,21419,428],{"className":21420},[109,113,146],[29,21422,154],{"className":21423},[153],[29,21425,21427],{"className":21426},[126],[29,21428,21430],{"className":21429,"style":161},[130],[29,21431],{},[29,21433,21435,21438],{"className":21434},[109],[29,21436,428],{"className":21437},[109,113],[29,21439,21441],{"className":21440},[117],[29,21442,21444,21473],{"className":21443},[121,122],[29,21445,21447,21470],{"className":21446},[126],[29,21448,21450],{"className":21449,"style":131},[130],[29,21451,21452,21455],{"style":134},[29,21453],{"className":21454,"style":139},[138],[29,21456,21458],{"className":21457},[143,144,145,146],[29,21459,21461,21464,21467],{"className":21460},[109,146],[29,21462,618],{"className":21463},[109,113,146],[29,21465,12362],{"className":21466},[3262,146],[29,21468,57],{"className":21469},[109,146],[29,21471,154],{"className":21472},[153],[29,21474,21476],{"className":21475},[126],[29,21477,21479],{"className":21478,"style":13026},[130],[29,21480],{},[29,21482],{"className":21483,"style":3258},[171],[29,21485,3236],{"className":21486},[3262],[29,21488],{"className":21489,"style":3258},[171],[29,21491,21493,21496,21499],{"className":21492},[100],[29,21494],{"className":21495,"style":479},[104],[29,21497,14576],{"className":21498},[109,113],[29,21500,466],{"className":21501},[490],[10,21503,21504],{},"过去全部信息压进一个固定大小的状态。不保存 KV，只保存状态。代价是长期记忆很差，而且不能并行。LSTM \u002F GRU 用门控改善了记忆，但没根治，也仍然不能并行。然后 Linear Attention 出现了：",[29,21506,21508],{"className":21507},[597],[29,21509,21511,21567],{"className":21510},[32],[29,21512,21514],{"className":21513},[36],[38,21515,21516],{"xmlns":40,"display":606},[42,21517,21518,21564],{},[45,21519,21520,21527,21529,21541,21543,21546,21548,21554,21556],{},[48,21521,21522,21525],{},[51,21523,21524],{},"S",[51,21526,618],{},[59,21528,508],{},[48,21530,21531,21533],{},[51,21532,21524],{},[45,21534,21535,21537,21539],{},[51,21536,618],{},[59,21538,12362],{},[55,21540,57],{},[59,21542,3236],{},[51,21544,21545],{},"ϕ",[59,21547,460],{"stretchy":459},[48,21549,21550,21552],{},[51,21551,1045],{},[51,21553,618],{},[59,21555,466],{"stretchy":459},[625,21557,21558,21560,21562],{},[51,21559,1067],{},[51,21561,618],{},[51,21563,1144],{"mathvariant":1002},[89,21565,21566],{"encoding":91},"S_t = S_{t-1} + \\phi(k_t) v_t^\\top",[29,21568,21570,21627,21692],{"className":21569,"ariaHidden":61},[96],[29,21571,21573,21576,21618,21621,21624],{"className":21572},[100],[29,21574],{"className":21575,"style":4046},[104],[29,21577,21579,21583],{"className":21578},[109],[29,21580,21524],{"className":21581,"style":21582},[109,113],"margin-right:0.0576em;",[29,21584,21586],{"className":21585},[117],[29,21587,21589,21610],{"className":21588},[121,122],[29,21590,21592,21607],{"className":21591},[126],[29,21593,21595],{"className":21594,"style":711},[130],[29,21596,21598,21601],{"style":21597},"top:-2.55em;margin-left:-0.0576em;margin-right:0.05em;",[29,21599],{"className":21600,"style":139},[138],[29,21602,21604],{"className":21603},[143,144,145,146],[29,21605,618],{"className":21606},[109,113,146],[29,21608,154],{"className":21609},[153],[29,21611,21613],{"className":21612},[126],[29,21614,21616],{"className":21615,"style":161},[130],[29,21617],{},[29,21619],{"className":21620,"style":367},[171],[29,21622,508],{"className":21623},[371],[29,21625],{"className":21626,"style":367},[171],[29,21628,21630,21634,21683,21686,21689],{"className":21629},[100],[29,21631],{"className":21632,"style":21633},[104],"height:0.8917em;vertical-align:-0.2083em;",[29,21635,21637,21640],{"className":21636},[109],[29,21638,21524],{"className":21639,"style":21582},[109,113],[29,21641,21643],{"className":21642},[117],[29,21644,21646,21675],{"className":21645},[121,122],[29,21647,21649,21672],{"className":21648},[126],[29,21650,21652],{"className":21651,"style":131},[130],[29,21653,21654,21657],{"style":21597},[29,21655],{"className":21656,"style":139},[138],[29,21658,21660],{"className":21659},[143,144,145,146],[29,21661,21663,21666,21669],{"className":21662},[109,146],[29,21664,618],{"className":21665},[109,113,146],[29,21667,12362],{"className":21668},[3262,146],[29,21670,57],{"className":21671},[109,146],[29,21673,154],{"className":21674},[153],[29,21676,21678],{"className":21677},[126],[29,21679,21681],{"className":21680,"style":13026},[130],[29,21682],{},[29,21684],{"className":21685,"style":3258},[171],[29,21687,3236],{"className":21688},[3262],[29,21690],{"className":21691,"style":3258},[171],[29,21693,21695,21699,21702,21705,21745,21748],{"className":21694},[100],[29,21696],{"className":21697,"style":21698},[104],"height:1.1491em;vertical-align:-0.25em;",[29,21700,21545],{"className":21701},[109,113],[29,21703,460],{"className":21704},[483],[29,21706,21708,21711],{"className":21707},[109],[29,21709,1045],{"className":21710,"style":1387},[109,113],[29,21712,21714],{"className":21713},[117],[29,21715,21717,21737],{"className":21716},[121,122],[29,21718,21720,21734],{"className":21719},[126],[29,21721,21723],{"className":21722,"style":711},[130],[29,21724,21725,21728],{"style":9323},[29,21726],{"className":21727,"style":139},[138],[29,21729,21731],{"className":21730},[143,144,145,146],[29,21732,618],{"className":21733},[109,113,146],[29,21735,154],{"className":21736},[153],[29,21738,21740],{"className":21739},[126],[29,21741,21743],{"className":21742,"style":161},[130],[29,21744],{},[29,21746,466],{"className":21747},[490],[29,21749,21751,21754],{"className":21750},[109],[29,21752,1067],{"className":21753,"style":1319},[109,113],[29,21755,21757],{"className":21756},[117],[29,21758,21760,21792],{"className":21759},[121,122],[29,21761,21763,21789],{"className":21762},[126],[29,21764,21766,21778],{"className":21765,"style":2442},[130],[29,21767,21769,21772],{"style":21768},"top:-2.453em;margin-left:-0.0359em;margin-right:0.05em;",[29,21770],{"className":21771,"style":139},[138],[29,21773,21775],{"className":21774},[143,144,145,146],[29,21776,618],{"className":21777},[109,113,146],[29,21779,21780,21783],{"style":2378},[29,21781],{"className":21782,"style":139},[138],[29,21784,21786],{"className":21785},[143,144,145,146],[29,21787,1144],{"className":21788},[109,146],[29,21790,154],{"className":21791},[153],[29,21793,21795],{"className":21794},[126],[29,21796,21798],{"className":21797,"style":2955},[130],[29,21799],{},[29,21801,21803],{"className":21802},[597],[29,21804,21806,21848],{"className":21805},[32],[29,21807,21809],{"className":21808},[36],[38,21810,21811],{"xmlns":40,"display":606},[42,21812,21813,21845],{},[45,21814,21815,21821,21823,21825,21827,21833,21839],{},[48,21816,21817,21819],{},[51,21818,615],{},[51,21820,618],{},[59,21822,508],{},[51,21824,21545],{},[59,21826,460],{"stretchy":459},[48,21828,21829,21831],{},[51,21830,1028],{},[51,21832,618],{},[301,21834,21835,21837],{},[59,21836,466],{"stretchy":459},[51,21838,1144],{"mathvariant":1002},[48,21840,21841,21843],{},[51,21842,21524],{},[51,21844,618],{},[89,21846,21847],{"encoding":91},"o_t = \\phi(q_t)^\\top S_t",[29,21849,21851,21906],{"className":21850,"ariaHidden":61},[96],[29,21852,21854,21857,21897,21900,21903],{"className":21853},[100],[29,21855],{"className":21856,"style":692},[104],[29,21858,21860,21863],{"className":21859},[109],[29,21861,615],{"className":21862},[109,113],[29,21864,21866],{"className":21865},[117],[29,21867,21869,21889],{"className":21868},[121,122],[29,21870,21872,21886],{"className":21871},[126],[29,21873,21875],{"className":21874,"style":711},[130],[29,21876,21877,21880],{"style":134},[29,21878],{"className":21879,"style":139},[138],[29,21881,21883],{"className":21882},[143,144,145,146],[29,21884,618],{"className":21885},[109,113,146],[29,21887,154],{"className":21888},[153],[29,21890,21892],{"className":21891},[126],[29,21893,21895],{"className":21894,"style":161},[130],[29,21896],{},[29,21898],{"className":21899,"style":367},[171],[29,21901,508],{"className":21902},[371],[29,21904],{"className":21905,"style":367},[171],[29,21907,21909,21912,21915,21918,21958,21987],{"className":21908},[100],[29,21910],{"className":21911,"style":21698},[104],[29,21913,21545],{"className":21914},[109,113],[29,21916,460],{"className":21917},[483],[29,21919,21921,21924],{"className":21920},[109],[29,21922,1028],{"className":21923,"style":1319},[109,113],[29,21925,21927],{"className":21926},[117],[29,21928,21930,21950],{"className":21929},[121,122],[29,21931,21933,21947],{"className":21932},[126],[29,21934,21936],{"className":21935,"style":711},[130],[29,21937,21938,21941],{"style":9370},[29,21939],{"className":21940,"style":139},[138],[29,21942,21944],{"className":21943},[143,144,145,146],[29,21945,618],{"className":21946},[109,113,146],[29,21948,154],{"className":21949},[153],[29,21951,21953],{"className":21952},[126],[29,21954,21956],{"className":21955,"style":161},[130],[29,21957],{},[29,21959,21961,21964],{"className":21960},[490],[29,21962,466],{"className":21963},[490],[29,21965,21967],{"className":21966},[117],[29,21968,21970],{"className":21969},[121],[29,21971,21973],{"className":21972},[126],[29,21974,21976],{"className":21975,"style":2442},[130],[29,21977,21978,21981],{"style":2378},[29,21979],{"className":21980,"style":139},[138],[29,21982,21984],{"className":21983},[143,144,145,146],[29,21985,1144],{"className":21986},[109,146],[29,21988,21990,21993],{"className":21989},[109],[29,21991,21524],{"className":21992,"style":21582},[109,113],[29,21994,21996],{"className":21995},[117],[29,21997,21999,22019],{"className":21998},[121,122],[29,22000,22002,22016],{"className":22001},[126],[29,22003,22005],{"className":22004,"style":711},[130],[29,22006,22007,22010],{"style":21597},[29,22008],{"className":22009,"style":139},[138],[29,22011,22013],{"className":22012},[143,144,145,146],[29,22014,618],{"className":22015},[109,113,146],[29,22017,154],{"className":22018},[153],[29,22020,22022],{"className":22021},[126],[29,22023,22025],{"className":22024,"style":161},[130],[29,22026],{},[10,22028,22029,22030,22073,22074,22077,22078,413],{},"每步只更新一个 state，复杂度 ",[29,22031,22033,22052],{"className":22032},[32],[29,22034,22036],{"className":22035},[36],[38,22037,22038],{"xmlns":40},[42,22039,22040,22050],{},[45,22041,22042,22044,22046,22048],{},[51,22043,4436],{},[59,22045,460],{"stretchy":459},[51,22047,3534],{},[59,22049,466],{"stretchy":459},[89,22051,21023],{"encoding":91},[29,22053,22055],{"className":22054,"ariaHidden":61},[96],[29,22056,22058,22061,22064,22067,22070],{"className":22057},[100],[29,22059],{"className":22060,"style":479},[104],[29,22062,4436],{"className":22063,"style":4465},[109,113],[29,22065,460],{"className":22066},[483],[29,22068,3534],{"className":22069,"style":3667},[109,113],[29,22071,466],{"className":22072},[490],"。看到这个式子会觉得眼熟：它就是 ",[14,22075,22076],{},"LSTM 式的 recurrent state"," 加 ",[14,22079,22080],{},"Attention 式的 query-key retrieval",[10,22082,22083,22084,22087,22088,22158,22159,22292,22293,22321],{},"所以它像一个\"现代 LSTM\"是有道理的，不是公式恰巧长得像。它重新选择了",[14,22085,22086],{},"压缩历史，而不是保存历史","。代价是信息碰撞。不同的 key 的信息全挤进同一个 state，容量有限，互相干扰。state ",[29,22089,22091,22109],{"className":22090},[32],[29,22092,22094],{"className":22093},[36],[38,22095,22096],{"xmlns":40},[42,22097,22098,22106],{},[45,22099,22100],{},[48,22101,22102,22104],{},[51,22103,21524],{},[51,22105,618],{},[89,22107,22108],{"encoding":91},"S_t",[29,22110,22112],{"className":22111,"ariaHidden":61},[96],[29,22113,22115,22118],{"className":22114},[100],[29,22116],{"className":22117,"style":4046},[104],[29,22119,22121,22124],{"className":22120},[109],[29,22122,21524],{"className":22123,"style":21582},[109,113],[29,22125,22127],{"className":22126},[117],[29,22128,22130,22150],{"className":22129},[121,122],[29,22131,22133,22147],{"className":22132},[126],[29,22134,22136],{"className":22135,"style":711},[130],[29,22137,22138,22141],{"style":21597},[29,22139],{"className":22140,"style":139},[138],[29,22142,22144],{"className":22143},[143,144,145,146],[29,22145,618],{"className":22146},[109,113,146],[29,22148,154],{"className":22149},[153],[29,22151,22153],{"className":22152},[126],[29,22154,22156],{"className":22155,"style":161},[130],[29,22157],{}," 的大小是固定的，跟 ",[29,22160,22162,22188],{"className":22161},[32],[29,22163,22165],{"className":22164},[36],[38,22166,22167],{"xmlns":40},[42,22168,22169,22185],{},[45,22170,22171,22177,22179],{},[48,22172,22173,22175],{},[51,22174,309],{},[51,22176,1045],{},[59,22178,3514],{},[48,22180,22181,22183],{},[51,22182,309],{},[51,22184,1067],{},[89,22186,22187],{"encoding":91},"d_k \\times d_v",[29,22189,22191,22246],{"className":22190,"ariaHidden":61},[96],[29,22192,22194,22197,22237,22240,22243],{"className":22193},[100],[29,22195],{"className":22196,"style":3098},[104],[29,22198,22200,22203],{"className":22199},[109],[29,22201,309],{"className":22202},[109,113],[29,22204,22206],{"className":22205},[117],[29,22207,22209,22229],{"className":22208},[121,122],[29,22210,22212,22226],{"className":22211},[126],[29,22213,22215],{"className":22214,"style":253},[130],[29,22216,22217,22220],{"style":134},[29,22218],{"className":22219,"style":139},[138],[29,22221,22223],{"className":22222},[143,144,145,146],[29,22224,1045],{"className":22225,"style":1387},[109,113,146],[29,22227,154],{"className":22228},[153],[29,22230,22232],{"className":22231},[126],[29,22233,22235],{"className":22234,"style":161},[130],[29,22236],{},[29,22238],{"className":22239,"style":3258},[171],[29,22241,3514],{"className":22242},[3262],[29,22244],{"className":22245,"style":3258},[171],[29,22247,22249,22252],{"className":22248},[100],[29,22250],{"className":22251,"style":3098},[104],[29,22253,22255,22258],{"className":22254},[109],[29,22256,309],{"className":22257},[109,113],[29,22259,22261],{"className":22260},[117],[29,22262,22264,22284],{"className":22263},[121,122],[29,22265,22267,22281],{"className":22266},[126],[29,22268,22270],{"className":22269,"style":3172},[130],[29,22271,22272,22275],{"style":134},[29,22273],{"className":22274,"style":139},[138],[29,22276,22278],{"className":22277},[143,144,145,146],[29,22279,1067],{"className":22280,"style":1319},[109,113,146],[29,22282,154],{"className":22283},[153],[29,22285,22287],{"className":22286},[126],[29,22288,22290],{"className":22289,"style":161},[130],[29,22291],{}," 同阶，不随 ",[29,22294,22296,22309],{"className":22295},[32],[29,22297,22299],{"className":22298},[36],[38,22300,22301],{"xmlns":40},[42,22302,22303,22307],{},[45,22304,22305],{},[51,22306,3534],{},[89,22308,3534],{"encoding":91},[29,22310,22312],{"className":22311,"ariaHidden":61},[96],[29,22313,22315,22318],{"className":22314},[100],[29,22316],{"className":22317,"style":3718},[104],[29,22319,3534],{"className":22320,"style":3667},[109,113]," 增长，而它要装的是全部历史的信息。序列越长，需要塞进去的东西越多，碰撞就越严重。具体表现是：想精确取回某一个早先 token 的信息时，取回来的其实是很多 token 叠加后的结果；full attention 能精确地只挑那一个，Linear Attention 做不到。门控能缓解遗忘，缓解不了容量上限造成的记混。",[10,22323,22324,22325,22399,22400,22443],{},"SSM（State Space Model）是这条线的另一个分支，典型代表是 Mamba。它和 Linear Attention 的数学形式不同，但目标一样：用固定大小的状态代替完整历史，把 ",[29,22326,22328,22352],{"className":22327},[32],[29,22329,22331],{"className":22330},[36],[38,22332,22333],{"xmlns":40},[42,22334,22335,22349],{},[45,22336,22337,22339,22341,22347],{},[51,22338,4436],{},[59,22340,460],{"stretchy":459},[301,22342,22343,22345],{},[51,22344,3534],{},[55,22346,69],{},[59,22348,466],{"stretchy":459},[89,22350,22351],{"encoding":91},"O(N^2)",[29,22353,22355],{"className":22354,"ariaHidden":61},[96],[29,22356,22358,22361,22364,22367,22396],{"className":22357},[100],[29,22359],{"className":22360,"style":4461},[104],[29,22362,4436],{"className":22363,"style":4465},[109,113],[29,22365,460],{"className":22366},[483],[29,22368,22370,22373],{"className":22369},[109],[29,22371,3534],{"className":22372,"style":3667},[109,113],[29,22374,22376],{"className":22375},[117],[29,22377,22379],{"className":22378},[121],[29,22380,22382],{"className":22381},[126],[29,22383,22385],{"className":22384,"style":4490},[130],[29,22386,22387,22390],{"style":403},[29,22388],{"className":22389,"style":139},[138],[29,22391,22393],{"className":22392},[143,144,145,146],[29,22394,69],{"className":22395},[109,146],[29,22397,466],{"className":22398},[490]," 降到 ",[29,22401,22403,22422],{"className":22402},[32],[29,22404,22406],{"className":22405},[36],[38,22407,22408],{"xmlns":40},[42,22409,22410,22420],{},[45,22411,22412,22414,22416,22418],{},[51,22413,4436],{},[59,22415,460],{"stretchy":459},[51,22417,3534],{},[59,22419,466],{"stretchy":459},[89,22421,21023],{"encoding":91},[29,22423,22425],{"className":22424,"ariaHidden":61},[96],[29,22426,22428,22431,22434,22437,22440],{"className":22427},[100],[29,22429],{"className":22430,"style":479},[104],[29,22432,4436],{"className":22433,"style":4465},[109,113],[29,22435,460],{"className":22436},[483],[29,22438,3534],{"className":22439,"style":3667},[109,113],[29,22441,466],{"className":22442},[490],"。Mamba 的特点在状态更新的方式上。它让状态转移矩阵随输入变化，相当于给状态加了一层输入相关的门控。这和 LSTM 的门控思路一脉相承，只是换了个数学外衣。",[10,22445,22446,22447,22450],{},"所以往上看，Mamba、RWKV、RetNet、DeltaNet、Gated DeltaNet、Kimi Linear，看起来像一堆论文名字动物园，其实都在回答同一个问题：",[14,22448,22449],{},"怎么用固定大小的记忆，尽可能少地丢掉历史。"," 于是有了 Hybrid：大部分层用廉价的 state memory，少数几层保留对完整历史的随机访问。这不是\"RNN 又赢了\"：attention 的随机访问能力和 RNN 的固定状态成本各有好处，把两者揉到一起，比宗教战争式地二选一更有产出。",[21,22452,22454],{"id":22453},"kimi-与-deepseek","Kimi 与 DeepSeek",[10,22456,22457,22458,413],{},"Kimi 和 DeepSeek 都在压历史，但压完之后的处理方式不一样：",[14,22459,22460],{},"Kimi 把历史压成 state，DeepSeek 压缩之后仍然保持可检索",[20081,22462,22464],{"id":22463},"state-machine-与-indexed-memory","State Machine 与 Indexed Memory",[10,22466,22467],{},"Kimi 的 Linear Attention 路线（KDA 那一支）是把历史递推成一个 state，查询时直接对 state 做操作：",[22469,22470,22475],"pre",{"className":22471,"code":22473,"language":3686,"meta":22474},[22472],"language-text","历史\n ↓\nrecurrent state\n ↓\nquery state\n","",[22476,22477,22473],"code",{"__ignoreMap":22474},[10,22479,22480],{},"DeepSeek 的路线是压缩，但压缩后的东西仍然带索引、可以被检索回来：",[22469,22482,22485],{"className":22483,"code":22484,"language":3686,"meta":22474},[22472],"历史\n ↓\ncompressed memory\n ↓\nindex \u002F retrieval\n ↓\nsparse attention\n",[22476,22486,22484],{"__ignoreMap":22474},[10,22488,22489,22490,22493,22494,22497,22498,22501],{},"我把它们概括成 ",[14,22491,22492],{},"State Machine vs. Indexed Memory","。这比说\"Linear Attention vs Sparse Attention\"有解释力：后者的区别看起来只是稀疏程度不同，前者是两种根本不同的 memory 哲学，一个",[14,22495,22496],{},"把历史变成状态","，一个",[14,22499,22500],{},"把历史变成可检索的索引","。有一个细节能说明 Kimi 这边并不只是\"换了个 attention\"。",[10,22503,22504],{},"K3 的技术报告把 refined training and data recipes、general\u002Fagentic\u002Fcoding RL、million-token agentic RL infrastructure、sandbox state，和 KDA、AttnRes、LatentMoE 并列成核心进展。注意这个列表的构成：一半是架构，一半是数据和基础设施。",[10,22506,22507],{},"如果 KDA 真是能力跃迁的唯一来源，报告没必要把 RL infra 抬到同样的位置。",[10,22509,22510,22511,22548,22549,22552,22553,22556],{},"反过来看，这也解释了 KDA 这类工作为什么重要。它让 ",[29,22512,22514,22530],{"className":22513},[32],[29,22515,22517],{"className":22516},[36],[38,22518,22519],{"xmlns":40},[42,22520,22521,22527],{},[45,22522,22523,22525],{},[55,22524,57],{},[76,22526,3849],{},[89,22528,22529],{"encoding":91},"1\\text{M}",[29,22531,22533],{"className":22532,"ariaHidden":61},[96],[29,22534,22536,22539,22542],{"className":22535},[100],[29,22537],{"className":22538,"style":3718},[104],[29,22540,57],{"className":22541},[109],[29,22543,22545],{"className":22544},[109,3686],[29,22546,3849],{"className":22547},[109]," context 和 long-horizon rollout 在计算上变得现实，于是以前喂不起的 RL trajectories 变得喂得起。架构在这里是",[14,22550,22551],{},"开路的","，不是",[14,22554,22555],{},"发力的","。DeepSeek 这边也一样，CSA\u002FHCA 让 1M context 变得足够便宜，价值主要落在这里。",[10,22558,22559,22560,22603,22604,22653,22654,22657],{},"那这两条路线谁能赢？我觉得决胜点不在复杂度写 ",[29,22561,22563,22582],{"className":22562},[32],[29,22564,22566],{"className":22565},[36],[38,22567,22568],{"xmlns":40},[42,22569,22570,22580],{},[45,22571,22572,22574,22576,22578],{},[51,22573,4436],{},[59,22575,460],{"stretchy":459},[51,22577,3534],{},[59,22579,466],{"stretchy":459},[89,22581,21023],{"encoding":91},[29,22583,22585],{"className":22584,"ariaHidden":61},[96],[29,22586,22588,22591,22594,22597,22600],{"className":22587},[100],[29,22589],{"className":22590,"style":479},[104],[29,22592,4436],{"className":22593,"style":4465},[109,113],[29,22595,460],{"className":22596},[483],[29,22598,3534],{"className":22599,"style":3667},[109,113],[29,22601,466],{"className":22602},[490]," 还是 ",[29,22605,22607,22629],{"className":22606},[32],[29,22608,22610],{"className":22609},[36],[38,22611,22612],{"xmlns":40},[42,22613,22614,22626],{},[45,22615,22616,22618,22620,22622,22624],{},[51,22617,4436],{},[59,22619,460],{"stretchy":459},[51,22621,3534],{},[51,22623,15311],{},[59,22625,466],{"stretchy":459},[89,22627,22628],{"encoding":91},"O(NB)",[29,22630,22632],{"className":22631,"ariaHidden":61},[96],[29,22633,22635,22638,22641,22644,22647,22650],{"className":22634},[100],[29,22636],{"className":22637,"style":479},[104],[29,22639,4436],{"className":22640,"style":4465},[109,113],[29,22642,460],{"className":22643},[483],[29,22645,3534],{"className":22646,"style":3667},[109,113],[29,22648,15311],{"className":22649,"style":17886},[109,113],[29,22651,466],{"className":22652},[490],"，而在更朴素的地方：",[14,22655,22656],{},"同样的显存带宽和 FLOPs 下，谁还能准确找回 50 万 token 前那个关键细节","。这是长上下文模型最难在 benchmark 上糊弄过去的地方。",[21,22659,22661],{"id":22660},"l-n-d","L × N × D",[10,22663,22664,22665,7323,22739,3722,22767,22795,22796,22824,22825,22853,22854,22882,22883,22911],{},"把前面的方案合起来看，它们动的其实是同一个三维对象，也就是 KV 的形状 ",[29,22666,22668,22691],{"className":22667},[32],[29,22669,22671],{"className":22670},[36],[38,22672,22673],{"xmlns":40},[42,22674,22675,22688],{},[45,22676,22677,22679,22681,22683,22685],{},[51,22678,3517],{},[59,22680,3514],{},[51,22682,3534],{},[59,22684,3514],{},[51,22686,22687],{},"D",[89,22689,22690],{"encoding":91},"L \\times N \\times D",[29,22692,22694,22712,22730],{"className":22693,"ariaHidden":61},[96],[29,22695,22697,22700,22703,22706,22709],{"className":22696},[100],[29,22698],{"className":22699,"style":3571},[104],[29,22701,3517],{"className":22702},[109,113],[29,22704],{"className":22705,"style":3258},[171],[29,22707,3514],{"className":22708},[3262],[29,22710],{"className":22711,"style":3258},[171],[29,22713,22715,22718,22721,22724,22727],{"className":22714},[100],[29,22716],{"className":22717,"style":3571},[104],[29,22719,3534],{"className":22720,"style":3667},[109,113],[29,22722],{"className":22723,"style":3258},[171],[29,22725,3514],{"className":22726},[3262],[29,22728],{"className":22729,"style":3258},[171],[29,22731,22733,22736],{"className":22732},[100],[29,22734],{"className":22735,"style":3718},[104],[29,22737,22687],{"className":22738,"style":4465},[109,113],[29,22740,22742,22755],{"className":22741},[32],[29,22743,22745],{"className":22744},[36],[38,22746,22747],{"xmlns":40},[42,22748,22749,22753],{},[45,22750,22751],{},[51,22752,3517],{},[89,22754,3517],{"encoding":91},[29,22756,22758],{"className":22757,"ariaHidden":61},[96],[29,22759,22761,22764],{"className":22760},[100],[29,22762],{"className":22763,"style":3718},[104],[29,22765,3517],{"className":22766},[109,113],[29,22768,22770,22783],{"className":22769},[32],[29,22771,22773],{"className":22772},[36],[38,22774,22775],{"xmlns":40},[42,22776,22777,22781],{},[45,22778,22779],{},[51,22780,3534],{},[89,22782,3534],{"encoding":91},[29,22784,22786],{"className":22785,"ariaHidden":61},[96],[29,22787,22789,22792],{"className":22788},[100],[29,22790],{"className":22791,"style":3718},[104],[29,22793,3534],{"className":22794,"style":3667},[109,113]," 是序列长度，",[29,22797,22799,22812],{"className":22798},[32],[29,22800,22802],{"className":22801},[36],[38,22803,22804],{"xmlns":40},[42,22805,22806,22810],{},[45,22807,22808],{},[51,22809,22687],{},[89,22811,22687],{"encoding":91},[29,22813,22815],{"className":22814,"ariaHidden":61},[96],[29,22816,22818,22821],{"className":22817},[100],[29,22819],{"className":22820,"style":3718},[104],[29,22822,22687],{"className":22823,"style":4465},[109,113]," 是每个 token 存下来的表示宽度。整条 DeepSeek 路线可以重新写成：GQA\u002FMLA 压 ",[29,22826,22828,22841],{"className":22827},[32],[29,22829,22831],{"className":22830},[36],[38,22832,22833],{"xmlns":40},[42,22834,22835,22839],{},[45,22836,22837],{},[51,22838,22687],{},[89,22840,22687],{"encoding":91},[29,22842,22844],{"className":22843,"ariaHidden":61},[96],[29,22845,22847,22850],{"className":22846},[100],[29,22848],{"className":22849,"style":3718},[104],[29,22851,22687],{"className":22852,"style":4465},[109,113],"，V4 的 CSA\u002FHCA 压 ",[29,22855,22857,22870],{"className":22856},[32],[29,22858,22860],{"className":22859},[36],[38,22861,22862],{"xmlns":40},[42,22863,22864,22868],{},[45,22865,22866],{},[51,22867,3534],{},[89,22869,3534],{"encoding":91},[29,22871,22873],{"className":22872,"ariaHidden":61},[96],[29,22874,22876,22879],{"className":22875},[100],[29,22877],{"className":22878,"style":3718},[104],[29,22880,3534],{"className":22881,"style":3667},[109,113],"，V4.1 压 ",[29,22884,22886,22899],{"className":22885},[32],[29,22887,22889],{"className":22888},[36],[38,22890,22891],{"xmlns":40},[42,22892,22893,22897],{},[45,22894,22895],{},[51,22896,3517],{},[89,22898,3517],{"encoding":91},[29,22900,22902],{"className":22901,"ariaHidden":61},[96],[29,22903,22905,22908],{"className":22904},[100],[29,22906],{"className":22907,"style":3718},[104],[29,22909,3517],{"className":22910},[109,113]," 和计算路径。到 V4.1，问题变成了：",[10,22913,22914],{},[14,22915,22916],{},"既然历史已经压缩过一次，为什么每一层还要重复为它付费？",[10,22918,22919,22920,22948,22949,22977,22978,23006,23007,23010],{},"ds4.1f 采用的是一个新架构族：552B MoE，Causal-Encoder-Decoder，输入侧约 8B activated、输出侧约 16B activated。这个不对称的计算预算是关键。要理解为什么 8B 和 16B 是分开的两个数，得把那两个瓶颈各自的账算一遍。先定符号。",[29,22921,22923,22936],{"className":22922},[32],[29,22924,22926],{"className":22925},[36],[38,22927,22928],{"xmlns":40},[42,22929,22930,22934],{},[45,22931,22932],{},[51,22933,16524],{},[89,22935,16524],{"encoding":91},[29,22937,22939],{"className":22938,"ariaHidden":61},[96],[29,22940,22942,22945],{"className":22941},[100],[29,22943],{"className":22944,"style":3718},[104],[29,22946,16524],{"className":22947,"style":2350},[109,113]," 是参数量（MoE 里指 activated 部分），",[29,22950,22952,22965],{"className":22951},[32],[29,22953,22955],{"className":22954},[36],[38,22956,22957],{"xmlns":40},[42,22958,22959,22963],{},[45,22960,22961],{},[51,22962,14576],{},[89,22964,14576],{"encoding":91},[29,22966,22968],{"className":22967,"ariaHidden":61},[96],[29,22969,22971,22974],{"className":22970},[100],[29,22972],{"className":22973,"style":440},[104],[29,22975,14576],{"className":22976},[109,113]," 是每个参数的字节数，",[29,22979,22981,22994],{"className":22980},[32],[29,22982,22984],{"className":22983},[36],[38,22985,22986],{"xmlns":40},[42,22987,22988,22992],{},[45,22989,22990],{},[51,22991,3534],{},[89,22993,3534],{"encoding":91},[29,22995,22997],{"className":22996,"ariaHidden":61},[96],[29,22998,23000,23003],{"className":22999},[100],[29,23001],{"className":23002,"style":3718},[104],[29,23004,3534],{"className":23005,"style":3667},[109,113]," 是序列长度。",[14,23008,23009],{},"输入侧","面对的是长 prompt。",[10,23012,23013,23014,23053,23054,413],{},"1M 个 token 全部要走一遍编码器，计算量约 ",[29,23015,23017,23035],{"className":23016},[32],[29,23018,23020],{"className":23019},[36],[38,23021,23022],{"xmlns":40},[42,23023,23024,23032],{},[45,23025,23026,23028,23030],{},[55,23027,69],{},[51,23029,16524],{},[51,23031,3534],{},[89,23033,23034],{"encoding":91},"2PN",[29,23036,23038],{"className":23037,"ariaHidden":61},[96],[29,23039,23041,23044,23047,23050],{"className":23040},[100],[29,23042],{"className":23043,"style":3718},[104],[29,23045,69],{"className":23046},[109],[29,23048,16524],{"className":23049,"style":2350},[109,113],[29,23051,3534],{"className":23052,"style":3667},[109,113],"，而要读的权重只有 ",[29,23055,23057,23075],{"className":23056},[32],[29,23058,23060],{"className":23059},[36],[38,23061,23062],{"xmlns":40},[42,23063,23064,23072],{},[45,23065,23066,23068,23070],{},[51,23067,16524],{},[59,23069,3514],{},[51,23071,14576],{},[89,23073,23074],{"encoding":91},"P \\times b",[29,23076,23078,23096],{"className":23077,"ariaHidden":61},[96],[29,23079,23081,23084,23087,23090,23093],{"className":23080},[100],[29,23082],{"className":23083,"style":3571},[104],[29,23085,16524],{"className":23086,"style":2350},[109,113],[29,23088],{"className":23089,"style":3258},[171],[29,23091,3514],{"className":23092},[3262],[29,23094],{"className":23095,"style":3258},[171],[29,23097,23099,23102],{"className":23098},[100],[29,23100],{"className":23101,"style":440},[104],[29,23103,14576],{"className":23104},[109,113],[10,23106,23107,23135,23136,23164,23165,23168,23169,23172,23173,23206,23207,23257],{},[29,23108,23110,23123],{"className":23109},[32],[29,23111,23113],{"className":23112},[36],[38,23114,23115],{"xmlns":40},[42,23116,23117,23121],{},[45,23118,23119],{},[51,23120,3534],{},[89,23122,3534],{"encoding":91},[29,23124,23126],{"className":23125,"ariaHidden":61},[96],[29,23127,23129,23132],{"className":23128},[100],[29,23130],{"className":23131,"style":3718},[104],[29,23133,3534],{"className":23134,"style":3667},[109,113]," 到百万量级时，算术强度能到几千 FLOPs\u002Fbyte，早就超过了硬件的算力带宽比。这一侧完全被算力卡住。想在 prefill 上省钱，唯一有效的办法就是把 ",[29,23137,23139,23152],{"className":23138},[32],[29,23140,23142],{"className":23141},[36],[38,23143,23144],{"xmlns":40},[42,23145,23146,23150],{},[45,23147,23148],{},[51,23149,16524],{},[89,23151,16524],{"encoding":91},[29,23153,23155],{"className":23154,"ariaHidden":61},[96],[29,23156,23158,23161],{"className":23157},[100],[29,23159],{"className":23160,"style":3718},[104],[29,23162,16524],{"className":23163,"style":2350},[109,113]," 压下去。",[14,23166,23167],{},"8B 是这么来的。"," ",[14,23170,23171],{},"输出侧","面对的是逐个生成的 token，情况正好反过来。每生成一个 token，计算量只有 ",[29,23174,23176,23191],{"className":23175},[32],[29,23177,23179],{"className":23178},[36],[38,23180,23181],{"xmlns":40},[42,23182,23183,23189],{},[45,23184,23185,23187],{},[55,23186,69],{},[51,23188,16524],{},[89,23190,16847],{"encoding":91},[29,23192,23194],{"className":23193,"ariaHidden":61},[96],[29,23195,23197,23200,23203],{"className":23196},[100],[29,23198],{"className":23199,"style":3718},[104],[29,23201,69],{"className":23202},[109],[29,23204,16524],{"className":23205,"style":2350},[109,113],"，但要读两份东西：模型权重 ",[29,23208,23210,23227],{"className":23209},[32],[29,23211,23213],{"className":23212},[36],[38,23214,23215],{"xmlns":40},[42,23216,23217,23225],{},[45,23218,23219,23221,23223],{},[51,23220,16524],{},[59,23222,3514],{},[51,23224,14576],{},[89,23226,23074],{"encoding":91},[29,23228,23230,23248],{"className":23229,"ariaHidden":61},[96],[29,23231,23233,23236,23239,23242,23245],{"className":23232},[100],[29,23234],{"className":23235,"style":3571},[104],[29,23237,16524],{"className":23238,"style":2350},[109,113],[29,23240],{"className":23241,"style":3258},[171],[29,23243,3514],{"className":23244},[3262],[29,23246],{"className":23247,"style":3258},[171],[29,23249,23251,23254],{"className":23250},[100],[29,23252],{"className":23253,"style":440},[104],[29,23255,14576],{"className":23256},[109,113],"，加上 KV Cache。而模型权重是必须整体读一遍的。",[10,23259,23260],{},"拿 16B activated、BF16 存储来算：",[29,23262,23264],{"className":23263},[597],[29,23265,23267,23298],{"className":23266},[32],[29,23268,23270],{"className":23269},[36],[38,23271,23272],{"xmlns":40,"display":606},[42,23273,23274,23295],{},[45,23275,23276,23279,23281,23283,23285,23288,23290,23293],{},[55,23277,23278],{},"16",[76,23280,15311],{},[59,23282,3514],{},[55,23284,69],{},[76,23286,23287],{}," bytes",[59,23289,508],{},[55,23291,23292],{},"32",[76,23294,19334],{},[89,23296,23297],{"encoding":91},"16\\text{B} \\times 2\\ \\text{bytes} = 32\\ \\text{GB}",[29,23299,23301,23325,23352],{"className":23300,"ariaHidden":61},[96],[29,23302,23304,23307,23310,23316,23319,23322],{"className":23303},[100],[29,23305],{"className":23306,"style":3571},[104],[29,23308,23278],{"className":23309},[109],[29,23311,23313],{"className":23312},[109,3686],[29,23314,15311],{"className":23315},[109],[29,23317],{"className":23318,"style":3258},[171],[29,23320,3514],{"className":23321},[3262],[29,23323],{"className":23324,"style":3258},[171],[29,23326,23328,23331,23334,23337,23343,23346,23349],{"className":23327},[100],[29,23329],{"className":23330,"style":556},[104],[29,23332,69],{"className":23333},[109],[29,23335,5624],{"className":23336},[171],[29,23338,23340],{"className":23339},[109,3686],[29,23341,3539],{"className":23342},[109],[29,23344],{"className":23345,"style":367},[171],[29,23347,508],{"className":23348},[371],[29,23350],{"className":23351,"style":367},[171],[29,23353,23355,23358,23361,23364],{"className":23354},[100],[29,23356],{"className":23357,"style":3718},[104],[29,23359,23292],{"className":23360},[109],[29,23362,5624],{"className":23363},[171],[29,23365,23367],{"className":23366},[109,3686],[29,23368,19430],{"className":23369},[109],[10,23371,23372,23373,23454,23455,23483,23484],{},"每生成一个 token，光权重就要搬 32 GB。而这一个 token 的计算量是 ",[29,23374,23376,23400],{"className":23375},[32],[29,23377,23379],{"className":23378},[36],[38,23380,23381],{"xmlns":40},[42,23382,23383,23397],{},[45,23384,23385,23387,23389,23391,23393,23395],{},[55,23386,69],{},[59,23388,3514],{},[55,23390,23278],{},[76,23392,15311],{},[59,23394,508],{},[55,23396,23292],{},[89,23398,23399],{"encoding":91},"2 \\times 16\\text{B} = 32",[29,23401,23403,23421,23445],{"className":23402,"ariaHidden":61},[96],[29,23404,23406,23409,23412,23415,23418],{"className":23405},[100],[29,23407],{"className":23408,"style":3552},[104],[29,23410,69],{"className":23411},[109],[29,23413],{"className":23414,"style":3258},[171],[29,23416,3514],{"className":23417},[3262],[29,23419],{"className":23420,"style":3258},[171],[29,23422,23424,23427,23430,23436,23439,23442],{"className":23423},[100],[29,23425],{"className":23426,"style":3718},[104],[29,23428,23278],{"className":23429},[109],[29,23431,23433],{"className":23432},[109,3686],[29,23434,15311],{"className":23435},[109],[29,23437],{"className":23438,"style":367},[171],[29,23440,508],{"className":23441},[371],[29,23443],{"className":23444,"style":367},[171],[29,23446,23448,23451],{"className":23447},[100],[29,23449],{"className":23450,"style":3272},[104],[29,23452,23292],{"className":23453},[109]," GFLOPs。算术强度大约是 ",[29,23456,23458,23471],{"className":23457},[32],[29,23459,23461],{"className":23460},[36],[38,23462,23463],{"xmlns":40},[42,23464,23465,23469],{},[45,23466,23467],{},[55,23468,57],{},[89,23470,57],{"encoding":91},[29,23472,23474],{"className":23473,"ariaHidden":61},[96],[29,23475,23477,23480],{"className":23476},[100],[29,23478],{"className":23479,"style":3272},[104],[29,23481,57],{"className":23482},[109]," FLOP\u002Fbyte。现代 GPU 的算力带宽比是几十到几百 FLOP\u002Fbyte。差了将近两个数量级，意味着算力几乎全程在等内存，多算一点几乎不花钱。既然算力在空转等数据，就没有理由在生成端省参数量。",[14,23485,23486],{},"16B 是这么来的。",[3046,23488,23490,23493],{"title":23489},"简化与推断边界",[10,23491,23492],{},"上面按\"每步只生成一个 token\"算。speculative decoding 会一次验证多个候选 token，test-time scaling、parallel decoding 也会改变每步的计算和内存特征，但不改变主线：无论一步生成几个 token，权重和 KV Cache 该读的还是要读。speculative decoding 甚至更吃带宽，草稿模型本身要读一遍权重，验证阶段还要再读主模型。",[10,23494,23495],{},"8B\u002F16B 落在 prefill 和 decode 两侧、方向符合各自瓶颈，这是我的读法，官方没有给出设计理由。",[10,23497,23498],{},"Compute bound 的那一侧压计算，memory bound 的那一侧压不住，就把能力堆上去。传统 decoder-only LLM 对 prompt 里每个 token 和 decode token 用几乎同一套昂贵 backbone，等于强迫两个瓶颈完全不同的阶段共用一套参数。",[10,23500,23501,23502,23561],{},"如果输入有 ",[29,23503,23505,23525],{"className":23504},[32],[29,23506,23508],{"className":23507},[36],[38,23509,23510],{"xmlns":40},[42,23511,23512,23522],{},[45,23513,23514,23516,23518,23520],{},[51,23515,3534],{},[59,23517,508],{},[55,23519,57],{},[76,23521,3849],{},[89,23523,23524],{"encoding":91},"N = 1\\text{M}",[29,23526,23528,23546],{"className":23527,"ariaHidden":61},[96],[29,23529,23531,23534,23537,23540,23543],{"className":23530},[100],[29,23532],{"className":23533,"style":3718},[104],[29,23535,3534],{"className":23536,"style":3667},[109,113],[29,23538],{"className":23539,"style":367},[171],[29,23541,508],{"className":23542},[371],[29,23544],{"className":23545,"style":367},[171],[29,23547,23549,23552,23555],{"className":23548},[100],[29,23550],{"className":23551,"style":3718},[104],[29,23553,57],{"className":23554},[109],[29,23556,23558],{"className":23557},[109,3686],[29,23559,3849],{"className":23560},[109],"，那么即使 attention 已经 sparse 或 compressed，你依然需要对百万 token 做很多层计算。V4.1 攻击的不只是 attention matrix：",[29,23563,23565],{"className":23564},[597],[29,23566,23568,23590],{"className":23567},[32],[29,23569,23571],{"className":23570},[36],[38,23572,23573],{"xmlns":40,"display":606},[42,23574,23575,23587],{},[45,23576,23577,23580],{},[51,23578,23579],{},"Q",[301,23581,23582,23585],{},[51,23583,23584],{},"K",[51,23586,1144],{"mathvariant":1002},[89,23588,23589],{"encoding":91},"QK^\\top",[29,23591,23593],{"className":23592,"ariaHidden":61},[96],[29,23594,23596,23600,23603],{"className":23595},[100],[29,23597],{"className":23598,"style":23599},[104],"height:1.0935em;vertical-align:-0.1944em;",[29,23601,23579],{"className":23602},[109,113],[29,23604,23606,23609],{"className":23605},[109],[29,23607,23584],{"className":23608,"style":4053},[109,113],[29,23610,23612],{"className":23611},[117],[29,23613,23615],{"className":23614},[121],[29,23616,23618],{"className":23617},[126],[29,23619,23621],{"className":23620,"style":2442},[130],[29,23622,23623,23626],{"style":2378},[29,23624],{"className":23625,"style":139},[138],[29,23627,23629],{"className":23628},[143,144,145,146],[29,23630,1144],{"className":23631},[109,146],[10,23633,23634,23635,23693,23694,23772,23773,413,23851,23879,23880,23938],{},"还攻击整个长序列每层 FFN\u002FMoE 的计算。前面账单上的第二项一直被 ",[29,23636,23638,23655],{"className":23637},[32],[29,23639,23641],{"className":23640},[36],[38,23642,23643],{"xmlns":40},[42,23644,23645,23653],{},[45,23646,23647],{},[301,23648,23649,23651],{},[51,23650,3534],{},[55,23652,69],{},[89,23654,4826],{"encoding":91},[29,23656,23658],{"className":23657,"ariaHidden":61},[96],[29,23659,23661,23664],{"className":23660},[100],[29,23662],{"className":23663,"style":4490},[104],[29,23665,23667,23670],{"className":23666},[109],[29,23668,3534],{"className":23669,"style":3667},[109,113],[29,23671,23673],{"className":23672},[117],[29,23674,23676],{"className":23675},[121],[29,23677,23679],{"className":23678},[126],[29,23680,23682],{"className":23681,"style":4490},[130],[29,23683,23684,23687],{"style":403},[29,23685],{"className":23686,"style":139},[138],[29,23688,23690],{"className":23689},[143,144,145,146],[29,23691,69],{"className":23692},[109,146]," 的光环盖着：attention 那项是 ",[29,23695,23697,23722],{"className":23696},[32],[29,23698,23700],{"className":23699},[36],[38,23701,23702],{"xmlns":40},[42,23703,23704,23720],{},[45,23705,23706,23708,23710,23716,23718],{},[51,23707,4436],{},[59,23709,460],{"stretchy":459},[301,23711,23712,23714],{},[51,23713,3534],{},[55,23715,69],{},[51,23717,309],{},[59,23719,466],{"stretchy":459},[89,23721,4534],{"encoding":91},[29,23723,23725],{"className":23724,"ariaHidden":61},[96],[29,23726,23728,23731,23734,23737,23766,23769],{"className":23727},[100],[29,23729],{"className":23730,"style":4461},[104],[29,23732,4436],{"className":23733,"style":4465},[109,113],[29,23735,460],{"className":23736},[483],[29,23738,23740,23743],{"className":23739},[109],[29,23741,3534],{"className":23742,"style":3667},[109,113],[29,23744,23746],{"className":23745},[117],[29,23747,23749],{"className":23748},[121],[29,23750,23752],{"className":23751},[126],[29,23753,23755],{"className":23754,"style":4490},[130],[29,23756,23757,23760],{"style":403},[29,23758],{"className":23759,"style":139},[138],[29,23761,23763],{"className":23762},[143,144,145,146],[29,23764,69],{"className":23765},[109,146],[29,23767,309],{"className":23768},[109,113],[29,23770,466],{"className":23771},[490],"，FFN\u002FMoE 那项是 ",[29,23774,23776,23801],{"className":23775},[32],[29,23777,23779],{"className":23778},[36],[38,23780,23781],{"xmlns":40},[42,23782,23783,23799],{},[45,23784,23785,23787,23789,23791,23797],{},[51,23786,4436],{},[59,23788,460],{"stretchy":459},[51,23790,3534],{},[301,23792,23793,23795],{},[51,23794,309],{},[55,23796,69],{},[59,23798,466],{"stretchy":459},[89,23800,4451],{"encoding":91},[29,23802,23804],{"className":23803,"ariaHidden":61},[96],[29,23805,23807,23810,23813,23816,23819,23848],{"className":23806},[100],[29,23808],{"className":23809,"style":4461},[104],[29,23811,4436],{"className":23812,"style":4465},[109,113],[29,23814,460],{"className":23815},[483],[29,23817,3534],{"className":23818,"style":3667},[109,113],[29,23820,23822,23825],{"className":23821},[109],[29,23823,309],{"className":23824},[109,113],[29,23826,23828],{"className":23827},[117],[29,23829,23831],{"className":23830},[121],[29,23832,23834],{"className":23833},[126],[29,23835,23837],{"className":23836,"style":4490},[130],[29,23838,23839,23842],{"style":403},[29,23840],{"className":23841,"style":139},[138],[29,23843,23845],{"className":23844},[143,144,145,146],[29,23846,69],{"className":23847},[109,146],[29,23849,466],{"className":23850},[490],[29,23852,23854,23867],{"className":23853},[32],[29,23855,23857],{"className":23856},[36],[38,23858,23859],{"xmlns":40},[42,23860,23861,23865],{},[45,23862,23863],{},[51,23864,3534],{},[89,23866,3534],{"encoding":91},[29,23868,23870],{"className":23869,"ariaHidden":61},[96],[29,23871,23873,23876],{"className":23872},[100],[29,23874],{"className":23875,"style":3718},[104],[29,23877,3534],{"className":23878,"style":3667},[109,113]," 大到百万量级、",[29,23881,23883,23900],{"className":23882},[32],[29,23884,23886],{"className":23885},[36],[38,23887,23888],{"xmlns":40},[42,23889,23890,23898],{},[45,23891,23892],{},[301,23893,23894,23896],{},[51,23895,3534],{},[55,23897,69],{},[89,23899,4826],{"encoding":91},[29,23901,23903],{"className":23902,"ariaHidden":61},[96],[29,23904,23906,23909],{"className":23905},[100],[29,23907],{"className":23908,"style":4490},[104],[29,23910,23912,23915],{"className":23911},[109],[29,23913,3534],{"className":23914,"style":3667},[109,113],[29,23916,23918],{"className":23917},[117],[29,23919,23921],{"className":23920},[121],[29,23922,23924],{"className":23923},[126],[29,23925,23927],{"className":23926,"style":4490},[130],[29,23928,23929,23932],{"style":403},[29,23930],{"className":23931,"style":139},[138],[29,23933,23935],{"className":23934},[143,144,145,146],[29,23936,69],{"className":23937},[109,146]," 又被打下来之后，后者因为每一层都要对每个 token 算一遍，纹丝不动。",[10,23940,23941],{},"所以 V4.1 做 asymmetric compute，本质上是：attention 都已经省下来了，总不能剩下的网络继续拿同样预算烧百万 token。终于有人检查了完整账单，而不是只盯着最显眼的那一行。",[10,23943,23944,23945,23950],{},"省下来的成本是多少？ds4.1f 的技术报告（",[6339,23946,23949],{"href":23947,"rel":23948},"https:\u002F\u002Fwww.alphaxiv.org\u002Fabs\u002F2609.deepseek-v4-1-flash",[6343],"DeepSeek-V4.1-Flash: Pushing the Limits of KV Cache Compression","）给了具体的数。相比上一代 V4-Flash：",[15609,23952,23953,23967],{},[15612,23954,23955,23956,23959,23960,23963,23964],{},"常驻 ",[14,23957,23958],{},"HBM"," 的 global KV cache：",[14,23961,23962],{},"890 字节 \u002F token","，约为原来的 ",[14,23965,23966],{},"1\u002F4",[15612,23968,23969,23970,23973,23974],{},"落在 ",[14,23971,23972],{},"SSD 或 host memory"," 的 persistent KV cache：约为原来的 ",[14,23975,23976],{},"1\u002F8",[10,23978,23979,23980,24008,24009,24037,24038,24066],{},"报告把这些收益归因于\"entry size、sequence dimension、layer dimension 三个方向上的协同优化\"，落到 CSA2 上就是三个模式：投影让 head 之间共享信息（压 ",[29,23981,23983,23996],{"className":23982},[32],[29,23984,23986],{"className":23985},[36],[38,23987,23988],{"xmlns":40},[42,23989,23990,23994],{},[45,23991,23992],{},[51,23993,22687],{},[89,23995,22687],{"encoding":91},[29,23997,23999],{"className":23998,"ariaHidden":61},[96],[29,24000,24002,24005],{"className":24001},[100],[29,24003],{"className":24004,"style":3718},[104],[29,24006,22687],{"className":24007,"style":4465},[109,113],"）、把多个 token 合成一个 KV entry（压 ",[29,24010,24012,24025],{"className":24011},[32],[29,24013,24015],{"className":24014},[36],[38,24016,24017],{"xmlns":40},[42,24018,24019,24023],{},[45,24020,24021],{},[51,24022,3534],{},[89,24024,3534],{"encoding":91},[29,24026,24028],{"className":24027,"ariaHidden":61},[96],[29,24029,24031,24034],{"className":24030},[100],[29,24032],{"className":24033,"style":3718},[104],[29,24035,3534],{"className":24036,"style":3667},[109,113],"）、跨层复用 KV（压 ",[29,24039,24041,24054],{"className":24040},[32],[29,24042,24044],{"className":24043},[36],[38,24045,24046],{"xmlns":40},[42,24047,24048,24052],{},[45,24049,24050],{},[51,24051,3517],{},[89,24053,3517],{"encoding":91},[29,24055,24057],{"className":24056,"ariaHidden":61},[96],[29,24058,24060,24063],{"className":24059},[100],[29,24061],{"className":24062,"style":3718},[104],[29,24064,3517],{"className":24065},[109,113],"）。再加上主 KV cache 本身换成 FP4。",[10,24068,24069,24144],{},[14,24070,24071,24072,413],{},"这不就是前面的 ",[29,24073,24075,24096],{"className":24074},[32],[29,24076,24078],{"className":24077},[36],[38,24079,24080],{"xmlns":40},[42,24081,24082,24094],{},[45,24083,24084,24086,24088,24090,24092],{},[51,24085,3517],{},[59,24087,3514],{},[51,24089,3534],{},[59,24091,3514],{},[51,24093,22687],{},[89,24095,22690],{"encoding":91},[29,24097,24099,24117,24135],{"className":24098,"ariaHidden":61},[96],[29,24100,24102,24105,24108,24111,24114],{"className":24101},[100],[29,24103],{"className":24104,"style":3571},[104],[29,24106,3517],{"className":24107},[109,113],[29,24109],{"className":24110,"style":3258},[171],[29,24112,3514],{"className":24113},[3262],[29,24115],{"className":24116,"style":3258},[171],[29,24118,24120,24123,24126,24129,24132],{"className":24119},[100],[29,24121],{"className":24122,"style":3571},[104],[29,24124,3534],{"className":24125,"style":3667},[109,113],[29,24127],{"className":24128,"style":3258},[171],[29,24130,3514],{"className":24131},[3262],[29,24133],{"className":24134,"style":3258},[171],[29,24136,24138,24141],{"className":24137},[100],[29,24139],{"className":24140,"style":3718},[104],[29,24142,22687],{"className":24143,"style":4465},[109,113]," 前面那条\"压宽度 → 压长度 → 压深度\"的路线，在 ds4.1f 上是第一次三个方向同时动刀。",[10,24146,24147,24148,24176],{},"压 ",[29,24149,24151,24164],{"className":24150},[32],[29,24152,24154],{"className":24153},[36],[38,24155,24156],{"xmlns":40},[42,24157,24158,24162],{},[45,24159,24160],{},[51,24161,3517],{},[89,24163,3517],{"encoding":91},[29,24165,24167],{"className":24166,"ariaHidden":61},[96],[29,24168,24170,24173],{"className":24169},[100],[29,24171],{"className":24172,"style":3718},[104],[29,24174,3517],{"className":24175},[109,113]," 这一个方向前面一直只是断言，说清楚它得看 CSA2 的三种模式：",[15609,24178,24179,24185,24195],{},[15612,24180,24181,24184],{},[14,24182,24183],{},"Full","：这层自己算主 K、V，投影一个 indexer key，产出一组新的 Top-K 索引",[15612,24186,24187,24190,24191,24194],{},[14,24188,24189],{},"Reindex","：主 K、V 和 indexer key 都从前一层继承，只重算 indexer query，于是选出一组",[14,24192,24193],{},"不同","的 Top-K",[15612,24196,24197,24200],{},[14,24198,24199],{},"Reuse","：主 K、V 和 Top-K 索引全部继承，这层几乎不做索引计算",[10,24202,24203],{},"再配一个分层的 indexer：第一个 Full 层先圈出一个 block 级的候选池，后面的 Reindex 层只在这个池子里打分。于是每个 query 的索引开销被限住，context 是一万还是一百万都一样。",[10,24205,24206],{},"这就是\"每一层不再重复为同一份历史付费\"的具体做法：多数层复用索引，少数层重建。",[10,24208,24209],{},"持久层那 1\u002F8 靠的是另一个部署优化，叫 SWA Bounded Replay：局部窗口的 cache 不再常驻，请求恢复时只重放最近一小段就能近似重建，不必把整条序列重算一遍。",[10,24211,24212,24213,24279,24280,24582],{},"另外 CED 让 prefill 的复杂度从 ",[29,24214,24216,24240],{"className":24215},[32],[29,24217,24219],{"className":24218},[36],[38,24220,24221],{"xmlns":40},[42,24222,24223,24237],{},[45,24224,24225,24227,24229,24231,24233,24235],{},[51,24226,4436],{},[59,24228,460],{"stretchy":459},[51,24230,3534],{},[59,24232,16535],{},[51,24234,3517],{},[59,24236,466],{"stretchy":459},[89,24238,24239],{"encoding":91},"O(N \\cdot L)",[29,24241,24243,24267],{"className":24242,"ariaHidden":61},[96],[29,24244,24246,24249,24252,24255,24258,24261,24264],{"className":24245},[100],[29,24247],{"className":24248,"style":479},[104],[29,24250,4436],{"className":24251,"style":4465},[109,113],[29,24253,460],{"className":24254},[483],[29,24256,3534],{"className":24257,"style":3667},[109,113],[29,24259],{"className":24260,"style":3258},[171],[29,24262,16535],{"className":24263},[3262],[29,24265],{"className":24266,"style":3258},[171],[29,24268,24270,24273,24276],{"className":24269},[100],[29,24271],{"className":24272,"style":479},[104],[29,24274,3517],{"className":24275},[109,113],[29,24277,466],{"className":24278},[490]," 降到大约 ",[29,24281,24283,24328],{"className":24282},[32],[29,24284,24286],{"className":24285},[36],[38,24287,24288],{"xmlns":40},[42,24289,24290,24325],{},[45,24291,24292,24294,24296,24298,24300,24306,24308,24315,24317,24323],{},[51,24293,4436],{},[59,24295,460],{"stretchy":459},[51,24297,3534],{},[59,24299,16535],{},[1088,24301,24302,24304],{},[51,24303,3517],{},[55,24305,69],{},[59,24307,3236],{},[48,24309,24310,24312],{},[51,24311,1013],{},[76,24313,24314],{},"win",[59,24316,16535],{},[1088,24318,24319,24321],{},[51,24320,3517],{},[55,24322,69],{},[59,24324,466],{"stretchy":459},[89,24326,24327],{"encoding":91},"O(N \\cdot \\frac{L}{2} + n_{\\text{win}} \\cdot \\frac{L}{2})",[29,24329,24331,24355,24443,24505],{"className":24330,"ariaHidden":61},[96],[29,24332,24334,24337,24340,24343,24346,24349,24352],{"className":24333},[100],[29,24335],{"className":24336,"style":479},[104],[29,24338,4436],{"className":24339,"style":4465},[109,113],[29,24341,460],{"className":24342},[483],[29,24344,3534],{"className":24345,"style":3667},[109,113],[29,24347],{"className":24348,"style":3258},[171],[29,24350,16535],{"className":24351},[3262],[29,24353],{"className":24354,"style":3258},[171],[29,24356,24358,24362,24434,24437,24440],{"className":24357},[100],[29,24359],{"className":24360,"style":24361},[104],"height:1.2173em;vertical-align:-0.345em;",[29,24363,24365,24368,24431],{"className":24364},[109],[29,24366],{"className":24367},[483,1549],[29,24369,24371],{"className":24370},[1088],[29,24372,24374,24422],{"className":24373},[121,122],[29,24375,24377,24419],{"className":24376},[126],[29,24378,24381,24396,24404],{"className":24379,"style":24380},[130],"height:0.8723em;",[29,24382,24384,24387],{"style":24383},"top:-2.655em;",[29,24385],{"className":24386,"style":5417},[138],[29,24388,24390],{"className":24389},[143,144,145,146],[29,24391,24393],{"className":24392},[109,146],[29,24394,69],{"className":24395},[109,146],[29,24397,24398,24401],{"style":5426},[29,24399],{"className":24400,"style":5417},[138],[29,24402],{"className":24403,"style":1776},[1775],[29,24405,24407,24410],{"style":24406},"top:-3.394em;",[29,24408],{"className":24409,"style":5417},[138],[29,24411,24413],{"className":24412},[143,144,145,146],[29,24414,24416],{"className":24415},[109,146],[29,24417,3517],{"className":24418},[109,113,146],[29,24420,154],{"className":24421},[153],[29,24423,24425],{"className":24424},[126],[29,24426,24429],{"className":24427,"style":24428},[130],"height:0.345em;",[29,24430],{},[29,24432],{"className":24433},[490,1549],[29,24435],{"className":24436,"style":3258},[171],[29,24438,3236],{"className":24439},[3262],[29,24441],{"className":24442,"style":3258},[171],[29,24444,24446,24450,24496,24499,24502],{"className":24445},[100],[29,24447],{"className":24448,"style":24449},[104],"height:0.5945em;vertical-align:-0.15em;",[29,24451,24453,24456],{"className":24452},[109],[29,24454,1013],{"className":24455},[109,113],[29,24457,24459],{"className":24458},[117],[29,24460,24462,24488],{"className":24461},[121,122],[29,24463,24465,24485],{"className":24464},[126],[29,24466,24468],{"className":24467,"style":4128},[130],[29,24469,24470,24473],{"style":134},[29,24471],{"className":24472,"style":139},[138],[29,24474,24476],{"className":24475},[143,144,145,146],[29,24477,24479],{"className":24478},[109,146],[29,24480,24482],{"className":24481},[109,3686,146],[29,24483,24314],{"className":24484},[109,146],[29,24486,154],{"className":24487},[153],[29,24489,24491],{"className":24490},[126],[29,24492,24494],{"className":24493,"style":161},[130],[29,24495],{},[29,24497],{"className":24498,"style":3258},[171],[29,24500,16535],{"className":24501},[3262],[29,24503],{"className":24504,"style":3258},[171],[29,24506,24508,24511,24579],{"className":24507},[100],[29,24509],{"className":24510,"style":24361},[104],[29,24512,24514,24517,24576],{"className":24513},[109],[29,24515],{"className":24516},[483,1549],[29,24518,24520],{"className":24519},[1088],[29,24521,24523,24568],{"className":24522},[121,122],[29,24524,24526,24565],{"className":24525},[126],[29,24527,24529,24543,24551],{"className":24528,"style":24380},[130],[29,24530,24531,24534],{"style":24383},[29,24532],{"className":24533,"style":5417},[138],[29,24535,24537],{"className":24536},[143,144,145,146],[29,24538,24540],{"className":24539},[109,146],[29,24541,69],{"className":24542},[109,146],[29,24544,24545,24548],{"style":5426},[29,24546],{"className":24547,"style":5417},[138],[29,24549],{"className":24550,"style":1776},[1775],[29,24552,24553,24556],{"style":24406},[29,24554],{"className":24555,"style":5417},[138],[29,24557,24559],{"className":24558},[143,144,145,146],[29,24560,24562],{"className":24561},[109,146],[29,24563,3517],{"className":24564},[109,113,146],[29,24566,154],{"className":24567},[153],[29,24569,24571],{"className":24570},[126],[29,24572,24574],{"className":24573,"style":24428},[130],[29,24575],{},[29,24577],{"className":24578},[490,1549],[29,24580,466],{"className":24581},[490],"，长输入的计算量差不多减半。",[10,24584,24585,24586,24591],{},"那 890 字节 \u002F token 在真实硬件上意味着什么。拿 B200 算一遍（",[6339,24587,24590],{"href":24588,"rel":24589},"https:\u002F\u002Finferencex.semianalysis.com\u002Fchips\u002Fb200",[6343],"规格","）。",[10,24593,24594,24595,24598],{},"先说部署形态。552B 权重按 FP4 存是 276 GB，单卡 180 GB 装不下，所以要拆卡。",[14,24596,24597],{},"四卡（TP4）就够","：每卡 69 GB 权重，还剩 111 GB 放 KV。",[10,24600,24601,24602,24605],{},"下面算的都是",[14,24603,24604],{},"每张卡","的数字。TP4 之后每卡只承担四分之一，所以字面值和耗时都除以 4，算术强度不变。",[10,24607,24608],{},"先把精度定死，因为算力和字节数都随它变：",[14645,24610,24611,24623],{},[14648,24612,24613],{},[14651,24614,24615,24617,24620],{},[14654,24616],{},[14654,24618,24619],{},"精度",[14654,24621,24622],{},"依据",[14663,24624,24625,24635,24646],{},[14651,24626,24627,24629,24632],{},[14668,24628,3972],{},[14668,24630,24631],{},"FP4",[14668,24633,24634],{},"每参数 0.5 字节",[14651,24636,24637,24640,24643],{},[14668,24638,24639],{},"KV cache",[14668,24641,24642],{},"FP4（MXFP4）",[14668,24644,24645],{},"技术报告的 QAT 就是按 MXFP4 做的，890 字节 \u002F token 是这个精度下的值",[14651,24647,24648,24651,24653],{},[14668,24649,24650],{},"算力基准",[14668,24652,24631],{},[14668,24654,24655],{},"B200 dense 9000 TFLOP\u002Fs",[10,24657,24658],{},"三项同一把尺子，拐点和字节数才可比。",[10,24660,24661],{},"B200 的 BF16 \u002F FP8 \u002F FP4 dense 算力分别是 2250 \u002F 4500 \u002F 9000 TFLOP\u002Fs，HBM3e 带宽 8 TB\u002Fs。FP4 的拐点是：",[29,24663,24665],{"className":24664},[597],[29,24666,24668,24702],{"className":24667},[32],[29,24669,24671],{"className":24670},[36],[38,24672,24673],{"xmlns":40,"display":606},[42,24674,24675,24699],{},[45,24676,24677,24692,24694,24697],{},[1088,24678,24679,24686],{},[45,24680,24681,24684],{},[55,24682,24683],{},"9000",[76,24685,15463],{},[45,24687,24688,24690],{},[55,24689,6035],{},[76,24691,15471],{},[59,24693,508],{},[55,24695,24696],{},"1125",[76,24698,15479],{},[89,24700,24701],{"encoding":91},"\\frac{9000\\ \\text{TFLOPS}}{8\\ \\text{TB\u002Fs}} = 1125\\ \\text{FLOP\u002Fbyte}",[29,24703,24705,24800],{"className":24704,"ariaHidden":61},[96],[29,24706,24708,24711,24791,24794,24797],{"className":24707},[100],[29,24709],{"className":24710,"style":15492},[104],[29,24712,24714,24717,24788],{"className":24713},[109],[29,24715],{"className":24716},[483,1549],[29,24718,24720],{"className":24719},[1088],[29,24721,24723,24780],{"className":24722},[121,122],[29,24724,24726,24777],{"className":24725},[126],[29,24727,24729,24749,24757],{"className":24728,"style":15511},[130],[29,24730,24731,24734],{"style":5413},[29,24732],{"className":24733,"style":5417},[138],[29,24735,24737,24740,24743],{"className":24736},[109],[29,24738,6035],{"className":24739},[109],[29,24741,5624],{"className":24742},[171],[29,24744,24746],{"className":24745},[109,3686],[29,24747,15532],{"className":24748},[109],[29,24750,24751,24754],{"style":5426},[29,24752],{"className":24753,"style":5417},[138],[29,24755],{"className":24756,"style":1776},[1775],[29,24758,24759,24762],{"style":5435},[29,24760],{"className":24761,"style":5417},[138],[29,24763,24765,24768,24771],{"className":24764},[109],[29,24766,24683],{"className":24767},[109],[29,24769,5624],{"className":24770},[171],[29,24772,24774],{"className":24773},[109,3686],[29,24775,15561],{"className":24776},[109],[29,24778,154],{"className":24779},[153],[29,24781,24783],{"className":24782},[126],[29,24784,24786],{"className":24785,"style":15571},[130],[29,24787],{},[29,24789],{"className":24790},[490,1549],[29,24792],{"className":24793,"style":367},[171],[29,24795,508],{"className":24796},[371],[29,24798],{"className":24799,"style":367},[171],[29,24801,24803,24806,24809,24812],{"className":24802},[100],[29,24804],{"className":24805,"style":479},[104],[29,24807,24696],{"className":24808},[109],[29,24810,5624],{"className":24811},[171],[29,24813,24815],{"className":24814},[109,3686],[29,24816,15604],{"className":24817},[109],[10,24819,24820],{},"decode 侧每 token 要搬多少？16B activated，FP4 存储：",[29,24822,24824],{"className":24823},[597],[29,24825,24827,24856],{"className":24826},[32],[29,24828,24830],{"className":24829},[36],[38,24831,24832],{"xmlns":40,"display":606},[42,24833,24834,24853],{},[45,24835,24836,24838,24840,24842,24845,24847,24849,24851],{},[55,24837,23278],{},[76,24839,15311],{},[59,24841,3514],{},[55,24843,24844],{},"0.5",[76,24846,16335],{},[59,24848,508],{},[55,24850,6035],{},[76,24852,19334],{},[89,24854,24855],{"encoding":91},"16\\text{B} \\times 0.5\\ \\text{字节} = 8\\ \\text{GB}",[29,24857,24859,24883,24910],{"className":24858,"ariaHidden":61},[96],[29,24860,24862,24865,24868,24874,24877,24880],{"className":24861},[100],[29,24863],{"className":24864,"style":3571},[104],[29,24866,23278],{"className":24867},[109],[29,24869,24871],{"className":24870},[109,3686],[29,24872,15311],{"className":24873},[109],[29,24875],{"className":24876,"style":3258},[171],[29,24878,3514],{"className":24879},[3262],[29,24881],{"className":24882,"style":3258},[171],[29,24884,24886,24889,24892,24895,24901,24904,24907],{"className":24885},[100],[29,24887],{"className":24888,"style":3718},[104],[29,24890,24844],{"className":24891},[109],[29,24893,5624],{"className":24894},[171],[29,24896,24898],{"className":24897},[109,3686],[29,24899,16408],{"className":24900},[109,16407],[29,24902],{"className":24903,"style":367},[171],[29,24905,508],{"className":24906},[371],[29,24908],{"className":24909,"style":367},[171],[29,24911,24913,24916,24919,24922],{"className":24912},[100],[29,24914],{"className":24915,"style":3718},[104],[29,24917,6035],{"className":24918},[109],[29,24920,5624],{"className":24921},[171],[29,24923,24925],{"className":24924},[109,3686],[29,24926,19430],{"className":24927},[109],[10,24929,24930,24931,25040],{},"KV 那边，1M context 时是 ",[29,24932,24934,24963],{"className":24933},[32],[29,24935,24937],{"className":24936},[36],[38,24938,24939],{"xmlns":40},[42,24940,24941,24960],{},[45,24942,24943,24946,24948,24955,24957],{},[55,24944,24945],{},"890",[59,24947,3514],{},[301,24949,24950,24952],{},[55,24951,16930],{},[55,24953,24954],{},"6",[59,24956,3843],{},[55,24958,24959],{},"0.93",[89,24961,24962],{"encoding":91},"890 \\times 10^6 \\approx 0.93",[29,24964,24966,24984,25031],{"className":24965,"ariaHidden":61},[96],[29,24967,24969,24972,24975,24978,24981],{"className":24968},[100],[29,24970],{"className":24971,"style":3552},[104],[29,24973,24945],{"className":24974},[109],[29,24976],{"className":24977,"style":3258},[171],[29,24979,3514],{"className":24980},[3262],[29,24982],{"className":24983,"style":3258},[171],[29,24985,24987,24990,24993,25022,25025,25028],{"className":24986},[100],[29,24988],{"className":24989,"style":4490},[104],[29,24991,57],{"className":24992},[109],[29,24994,24996,24999],{"className":24995},[109],[29,24997,17058],{"className":24998},[109],[29,25000,25002],{"className":25001},[117],[29,25003,25005],{"className":25004},[121],[29,25006,25008],{"className":25007},[126],[29,25009,25011],{"className":25010,"style":4490},[130],[29,25012,25013,25016],{"style":403},[29,25014],{"className":25015,"style":139},[138],[29,25017,25019],{"className":25018},[143,144,145,146],[29,25020,24954],{"className":25021},[109,146],[29,25023],{"className":25024,"style":367},[171],[29,25026,3843],{"className":25027},[371],[29,25029],{"className":25030,"style":367},[171],[29,25032,25034,25037],{"className":25033},[100],[29,25035],{"className":25036,"style":3272},[104],[29,25038,24959],{"className":25039},[109]," GB。加起来 8.93 GB。算术强度：",[29,25042,25044],{"className":25043},[597],[29,25045,25047,25089],{"className":25046},[32],[29,25048,25050],{"className":25049},[36],[38,25051,25052],{"xmlns":40,"display":606},[42,25053,25054,25086],{},[45,25055,25056,25058,25060,25079,25081,25084],{},[51,25057,16474],{},[59,25059,508],{},[1088,25061,25062,25072],{},[45,25063,25064,25066,25068,25070],{},[55,25065,69],{},[59,25067,3514],{},[55,25069,23278],{},[76,25071,15311],{},[45,25073,25074,25077],{},[55,25075,25076],{},"8.93",[76,25078,19334],{},[59,25080,3843],{},[55,25082,25083],{},"3.6",[76,25085,15479],{},[89,25087,25088],{"encoding":91},"I = \\frac{2 \\times 16\\text{B}}{8.93\\ \\text{GB}} \\approx 3.6\\ \\text{FLOP\u002Fbyte}",[29,25090,25092,25110,25214],{"className":25091,"ariaHidden":61},[96],[29,25093,25095,25098,25101,25104,25107],{"className":25094},[100],[29,25096],{"className":25097,"style":3718},[104],[29,25099,16474],{"className":25100,"style":16489},[109,113],[29,25102],{"className":25103,"style":367},[171],[29,25105,508],{"className":25106},[371],[29,25108],{"className":25109,"style":367},[171],[29,25111,25113,25116,25205,25208,25211],{"className":25112},[100],[29,25114],{"className":25115,"style":20815},[104],[29,25117,25119,25122,25202],{"className":25118},[109],[29,25120],{"className":25121},[483,1549],[29,25123,25125],{"className":25124},[1088],[29,25126,25128,25194],{"className":25127},[121,122],[29,25129,25131,25191],{"className":25130},[126],[29,25132,25134,25154,25162],{"className":25133,"style":15511},[130],[29,25135,25136,25139],{"style":5413},[29,25137],{"className":25138,"style":5417},[138],[29,25140,25142,25145,25148],{"className":25141},[109],[29,25143,25076],{"className":25144},[109],[29,25146,5624],{"className":25147},[171],[29,25149,25151],{"className":25150},[109,3686],[29,25152,19430],{"className":25153},[109],[29,25155,25156,25159],{"style":5426},[29,25157],{"className":25158,"style":5417},[138],[29,25160],{"className":25161,"style":1776},[1775],[29,25163,25164,25167],{"style":5435},[29,25165],{"className":25166,"style":5417},[138],[29,25168,25170,25173,25176,25179,25182,25185],{"className":25169},[109],[29,25171,69],{"className":25172},[109],[29,25174],{"className":25175,"style":3258},[171],[29,25177,3514],{"className":25178},[3262],[29,25180],{"className":25181,"style":3258},[171],[29,25183,23278],{"className":25184},[109],[29,25186,25188],{"className":25187},[109,3686],[29,25189,15311],{"className":25190},[109],[29,25192,154],{"className":25193},[153],[29,25195,25197],{"className":25196},[126],[29,25198,25200],{"className":25199,"style":5457},[130],[29,25201],{},[29,25203],{"className":25204},[490,1549],[29,25206],{"className":25207,"style":367},[171],[29,25209,3843],{"className":25210},[371],[29,25212],{"className":25213,"style":367},[171],[29,25215,25217,25220,25223,25226],{"className":25216},[100],[29,25218],{"className":25219,"style":479},[104],[29,25221,25083],{"className":25222},[109],[29,25224,5624],{"className":25225},[171],[29,25227,25229],{"className":25228},[109,3686],[29,25230,15604],{"className":25231},[109],[10,25233,25234],{},"按 8 TB\u002Fs 的带宽上限，单流大约 900 token\u002Fs。",[10,25236,25237,25240],{},[14,25238,25239],{},"3.6 对 1125，差了 300 倍。"," B200 的 FP4 算力有 99.7% 用不上。",[10,25242,25243],{},"换权重精度结论也一样。KV 那 890 字节 \u002F token 已经是 FP4 下的实测值，所以只动权重：",[14645,25245,25246,25264],{},[14648,25247,25248],{},[14651,25249,25250,25253,25256,25259,25261],{},[14654,25251,25252],{},"权重精度",[14654,25254,25255],{},"权重字节 \u002F token",[14654,25257,25258],{},"拐点",[14654,25260,17772],{},[14654,25262,25263],{},"差几倍",[14663,25265,25266,25281,25298],{},[14651,25267,25268,25270,25273,25275,25278],{},[14668,25269,24631],{},[14668,25271,25272],{},"8.0 GB",[14668,25274,24696],{},[14668,25276,25277],{},"3.58",[14668,25279,25280],{},"314×",[14651,25282,25283,25286,25289,25292,25295],{},[14668,25284,25285],{},"FP8",[14668,25287,25288],{},"16.0 GB",[14668,25290,25291],{},"562",[14668,25293,25294],{},"1.89",[14668,25296,25297],{},"298×",[14651,25299,25300,25303,25306,25309,25312],{},[14668,25301,25302],{},"BF16",[14668,25304,25305],{},"32.0 GB",[14668,25307,25308],{},"281",[14668,25310,25311],{},"0.97",[14668,25313,25314],{},"289×",[10,25316,25317,25318,25481,25482,413],{},"精度降一半，拐点翻一倍，但算术强度也跟着掉一半，两边抵消。根源是 B200 上这三个精度的「算力 × 每参字节」都是同一个数：",[29,25319,25321,25361],{"className":25320},[32],[29,25322,25324],{"className":25323},[36],[38,25325,25326],{"xmlns":40},[42,25327,25328,25358],{},[45,25329,25330,25332,25334,25336,25338,25341,25343,25345,25347,25350,25352,25354,25356],{},[55,25331,24683],{},[59,25333,3514],{},[55,25335,24844],{},[59,25337,508],{},[55,25339,25340],{},"4500",[59,25342,3514],{},[55,25344,57],{},[59,25346,508],{},[55,25348,25349],{},"2250",[59,25351,3514],{},[55,25353,69],{},[59,25355,508],{},[55,25357,25340],{},[89,25359,25360],{"encoding":91},"9000 \\times 0.5 = 4500 \\times 1 = 2250 \\times 2 = 4500",[29,25362,25364,25382,25400,25418,25436,25454,25472],{"className":25363,"ariaHidden":61},[96],[29,25365,25367,25370,25373,25376,25379],{"className":25366},[100],[29,25368],{"className":25369,"style":3552},[104],[29,25371,24683],{"className":25372},[109],[29,25374],{"className":25375,"style":3258},[171],[29,25377,3514],{"className":25378},[3262],[29,25380],{"className":25381,"style":3258},[171],[29,25383,25385,25388,25391,25394,25397],{"className":25384},[100],[29,25386],{"className":25387,"style":3272},[104],[29,25389,24844],{"className":25390},[109],[29,25392],{"className":25393,"style":367},[171],[29,25395,508],{"className":25396},[371],[29,25398],{"className":25399,"style":367},[171],[29,25401,25403,25406,25409,25412,25415],{"className":25402},[100],[29,25404],{"className":25405,"style":3552},[104],[29,25407,25340],{"className":25408},[109],[29,25410],{"className":25411,"style":3258},[171],[29,25413,3514],{"className":25414},[3262],[29,25416],{"className":25417,"style":3258},[171],[29,25419,25421,25424,25427,25430,25433],{"className":25420},[100],[29,25422],{"className":25423,"style":3272},[104],[29,25425,57],{"className":25426},[109],[29,25428],{"className":25429,"style":367},[171],[29,25431,508],{"className":25432},[371],[29,25434],{"className":25435,"style":367},[171],[29,25437,25439,25442,25445,25448,25451],{"className":25438},[100],[29,25440],{"className":25441,"style":3552},[104],[29,25443,25349],{"className":25444},[109],[29,25446],{"className":25447,"style":3258},[171],[29,25449,3514],{"className":25450},[3262],[29,25452],{"className":25453,"style":3258},[171],[29,25455,25457,25460,25463,25466,25469],{"className":25456},[100],[29,25458],{"className":25459,"style":3272},[104],[29,25461,69],{"className":25462},[109],[29,25464],{"className":25465,"style":367},[171],[29,25467,508],{"className":25468},[371],[29,25470],{"className":25471,"style":367},[171],[29,25473,25475,25478],{"className":25474},[100],[29,25476],{"className":25477,"style":3272},[104],[29,25479,25340],{"className":25480},[109],"。所以",[14,25483,25484],{},"差多少倍基本由架构决定，不由精度决定",[10,25486,25487],{},"流量拆开看：",[14645,25489,25490,25502],{},[14648,25491,25492],{},[14651,25493,25494,25497,25499],{},[14654,25495,25496],{},"1M context 下每 token 的流量",[14654,25498,16408],{},[14654,25500,25501],{},"占比",[14663,25503,25504,25514],{},[14651,25505,25506,25509,25511],{},[14668,25507,25508],{},"权重",[14668,25510,25272],{},[14668,25512,25513],{},"90%",[14651,25515,25516,25519,25522],{},[14668,25517,25518],{},"KV",[14668,25520,25521],{},"0.93 GB",[14668,25523,25524],{},"10%",[10,25526,25527,25528,413],{},"模型结构和硬件的 co-design 之后，",[14,25529,25530],{},"KV 已经不是瓶颈",[10,25532,25533],{},"ds4.1f 把 KV 压到 890 字节 \u002F token 之后，就算跑到 1M context，它也只占总流量的 10%。再压一半，总时间只少 6%；压到四分之一，也只少 9%。",[10,25535,25536],{},"而权重占 90%。把它减半，就是接近一倍的提速。",[10,25538,25539],{},"字节数直接决定速度。decode 是 memory bound，每个 token 的耗时就是总字节除以带宽：",[14645,25541,25542,25589],{},[14648,25543,25544],{},[14651,25545,25546,25576,25578,25580,25583,25586],{},[14654,25547,25548],{},[29,25549,25551,25564],{"className":25550},[32],[29,25552,25554],{"className":25553},[36],[38,25555,25556],{"xmlns":40},[42,25557,25558,25562],{},[45,25559,25560],{},[51,25561,3534],{},[89,25563,3534],{"encoding":91},[29,25565,25567],{"className":25566,"ariaHidden":61},[96],[29,25568,25570,25573],{"className":25569},[100],[29,25571],{"className":25572,"style":3718},[104],[29,25574,3534],{"className":25575,"style":3667},[109,113],[14654,25577,25508],{},[14654,25579,25518],{},[14654,25581,25582],{},"合计",[14654,25584,25585],{},"每 token",[14654,25587,25588],{},"单流 tok\u002Fs",[14663,25590,25591,25609,25626,25644],{},[14651,25592,25593,25595,25598,25601,25603,25606],{},[14668,25594,17788],{},[14668,25596,25597],{},"8.00 GB",[14668,25599,25600],{},"0.004 GB",[14668,25602,25597],{},[14668,25604,25605],{},"1.00 ms",[14668,25607,25608],{},"1000",[14651,25610,25611,25613,25615,25618,25621,25623],{},[14668,25612,17805],{},[14668,25614,25597],{},[14668,25616,25617],{},"0.029 GB",[14668,25619,25620],{},"8.03 GB",[14668,25622,25605],{},[14668,25624,25625],{},"996",[14651,25627,25628,25630,25632,25635,25638,25641],{},[14668,25629,17821],{},[14668,25631,25597],{},[14668,25633,25634],{},"0.117 GB",[14668,25636,25637],{},"8.12 GB",[14668,25639,25640],{},"1.01 ms",[14668,25642,25643],{},"986",[14651,25645,25646,25648,25650,25653,25656,25659],{},[14668,25647,17837],{},[14668,25649,25597],{},[14668,25651,25652],{},"0.933 GB",[14668,25654,25655],{},"8.93 GB",[14668,25657,25658],{},"1.12 ms",[14668,25660,25661],{},"896",[10,25663,25664,25667],{},[14,25665,25666],{},"从 4k 到 1M，单流速度只掉 10%。"," KV 压缩买到的是跑长 context 不掉速，能跑起来只是附带的结果。",[10,25669,25670],{},"再看容量。单卡 172 GB 可用显存，按 890 字节 \u002F token：",[29,25672,25674],{"className":25673},[597],[29,25675,25677,25713],{"className":25676},[32],[29,25678,25680],{"className":25679},[36],[38,25681,25682],{"xmlns":40,"display":606},[42,25683,25684,25710],{},[45,25685,25686,25702,25704,25707],{},[1088,25687,25688,25695],{},[45,25689,25690,25693],{},[55,25691,25692],{},"172",[76,25694,19334],{},[45,25696,25697,25699],{},[55,25698,24945],{},[76,25700,25701],{}," B",[59,25703,3843],{},[55,25705,25706],{},"1.93",[76,25708,25709],{}," 亿 token",[89,25711,25712],{"encoding":91},"\\frac{172\\ \\text{GB}}{890\\ \\text{B}} \\approx 1.93\\ \\text{亿 token}",[29,25714,25716,25811],{"className":25715,"ariaHidden":61},[96],[29,25717,25719,25722,25802,25805,25808],{"className":25718},[100],[29,25720],{"className":25721,"style":20815},[104],[29,25723,25725,25728,25799],{"className":25724},[109],[29,25726],{"className":25727},[483,1549],[29,25729,25731],{"className":25730},[1088],[29,25732,25734,25791],{"className":25733},[121,122],[29,25735,25737,25788],{"className":25736},[126],[29,25738,25740,25760,25768],{"className":25739,"style":15511},[130],[29,25741,25742,25745],{"style":5413},[29,25743],{"className":25744,"style":5417},[138],[29,25746,25748,25751,25754],{"className":25747},[109],[29,25749,24945],{"className":25750},[109],[29,25752,5624],{"className":25753},[171],[29,25755,25757],{"className":25756},[109,3686],[29,25758,15311],{"className":25759},[109],[29,25761,25762,25765],{"style":5426},[29,25763],{"className":25764,"style":5417},[138],[29,25766],{"className":25767,"style":1776},[1775],[29,25769,25770,25773],{"style":5435},[29,25771],{"className":25772,"style":5417},[138],[29,25774,25776,25779,25782],{"className":25775},[109],[29,25777,25692],{"className":25778},[109],[29,25780,5624],{"className":25781},[171],[29,25783,25785],{"className":25784},[109,3686],[29,25786,19430],{"className":25787},[109],[29,25789,154],{"className":25790},[153],[29,25792,25794],{"className":25793},[126],[29,25795,25797],{"className":25796,"style":5457},[130],[29,25798],{},[29,25800],{"className":25801},[490,1549],[29,25803],{"className":25804,"style":367},[171],[29,25806,3843],{"className":25807},[371],[29,25809],{"className":25810,"style":367},[171],[29,25812,25814,25817,25820,25823],{"className":25813},[100],[29,25815],{"className":25816,"style":440},[104],[29,25818,25706],{"className":25819},[109],[29,25821,5624],{"className":25822},[171],[29,25824,25826,25830],{"className":25825},[109,3686],[29,25827,25829],{"className":25828},[109,16407],"亿",[29,25831,25833],{"className":25832},[109]," token",[10,25835,25836],{},"一张卡装得下 1.93 亿 token 的 global KV。",[10,25838,25839],{},"但容量不是硬上限。装不下就往下沉，host DRAM、SSD 一级一级放；请求再来时命中 prefix cache，把 KV 搬回 HBM 就能接着算。技术报告里那两个数本来就是按这个层级给的：常驻 HBM 的 global KV 是 890 字节 \u002F token，而落在 SSD 或 host memory 的 persistent KV 再压到 V4-Flash 的 1\u002F8。",[10,25841,25842],{},[14,25843,25844],{},"KV 这一层已经从\"装不下就崩\"的硬约束，变成了一个分层的存储问题。",[10,25846,25847],{},"所以这条线走到这里，战场又换了一次位置：从\"压 KV\"换到\"压权重\"。前面几代解决的是历史存成什么样，接下来要解决的是 16B activated 这个数本身。",[10,25849,25850],{},"batch 能不能救？批处理把权重摊薄，但同时把 KV 按 batch 复制：",[29,25852,25854],{"className":25853},[597],[29,25855,25857,25917],{"className":25856},[32],[29,25858,25860],{"className":25859},[36],[38,25861,25862],{"xmlns":40,"display":606},[42,25863,25864,25914],{},[45,25865,25866,25868,25870,25872,25874,25876,25878,25880],{},[51,25867,16474],{},[59,25869,460],{"stretchy":459},[51,25871,15311],{},[59,25873,62],{"separator":61},[51,25875,3534],{},[59,25877,466],{"stretchy":459},[59,25879,508],{},[1088,25881,25882,25896],{},[45,25883,25884,25886,25888,25890,25892,25894],{},[51,25885,15311],{},[59,25887,3514],{},[55,25889,69],{},[59,25891,3514],{},[55,25893,23278],{},[76,25895,15311],{},[45,25897,25898,25900,25902,25904,25906,25908,25910,25912],{},[55,25899,6035],{},[76,25901,19334],{},[59,25903,3236],{},[51,25905,15311],{},[59,25907,3514],{},[55,25909,24945],{},[59,25911,3514],{},[51,25913,3534],{},[89,25915,25916],{"encoding":91},"I(B, N) = \\frac{B \\times 2 \\times 16\\text{B}}{8\\ \\text{GB} + B \\times 890 \\times N}",[29,25918,25920,25956],{"className":25919,"ariaHidden":61},[96],[29,25921,25923,25926,25929,25932,25935,25938,25941,25944,25947,25950,25953],{"className":25922},[100],[29,25924],{"className":25925,"style":479},[104],[29,25927,16474],{"className":25928,"style":16489},[109,113],[29,25930,460],{"className":25931},[483],[29,25933,15311],{"className":25934,"style":17886},[109,113],[29,25936,62],{"className":25937},[167],[29,25939],{"className":25940,"style":172},[171],[29,25942,3534],{"className":25943,"style":3667},[109,113],[29,25945,466],{"className":25946},[490],[29,25948],{"className":25949,"style":367},[171],[29,25951,508],{"className":25952},[371],[29,25954],{"className":25955,"style":367},[171],[29,25957,25959,25963],{"className":25958},[100],[29,25960],{"className":25961,"style":25962},[104],"height:2.1297em;vertical-align:-0.7693em;",[29,25964,25966,25969,26097],{"className":25965},[109],[29,25967],{"className":25968},[483,1549],[29,25970,25972],{"className":25971},[1088],[29,25973,25975,26089],{"className":25974},[121,122],[29,25976,25978,26086],{"className":25977},[126],[29,25979,25981,26037,26045],{"className":25980,"style":15511},[130],[29,25982,25983,25986],{"style":5413},[29,25984],{"className":25985,"style":5417},[138],[29,25987,25989,25992,25995,26001,26004,26007,26010,26013,26016,26019,26022,26025,26028,26031,26034],{"className":25988},[109],[29,25990,6035],{"className":25991},[109],[29,25993,5624],{"className":25994},[171],[29,25996,25998],{"className":25997},[109,3686],[29,25999,19430],{"className":26000},[109],[29,26002],{"className":26003,"style":3258},[171],[29,26005,3236],{"className":26006},[3262],[29,26008],{"className":26009,"style":3258},[171],[29,26011,15311],{"className":26012,"style":17886},[109,113],[29,26014],{"className":26015,"style":3258},[171],[29,26017,3514],{"className":26018},[3262],[29,26020],{"className":26021,"style":3258},[171],[29,26023,24945],{"className":26024},[109],[29,26026],{"className":26027,"style":3258},[171],[29,26029,3514],{"className":26030},[3262],[29,26032],{"className":26033,"style":3258},[171],[29,26035,3534],{"className":26036,"style":3667},[109,113],[29,26038,26039,26042],{"style":5426},[29,26040],{"className":26041,"style":5417},[138],[29,26043],{"className":26044,"style":1776},[1775],[29,26046,26047,26050],{"style":5435},[29,26048],{"className":26049,"style":5417},[138],[29,26051,26053,26056,26059,26062,26065,26068,26071,26074,26077,26080],{"className":26052},[109],[29,26054,15311],{"className":26055,"style":17886},[109,113],[29,26057],{"className":26058,"style":3258},[171],[29,26060,3514],{"className":26061},[3262],[29,26063],{"className":26064,"style":3258},[171],[29,26066,69],{"className":26067},[109],[29,26069],{"className":26070,"style":3258},[171],[29,26072,3514],{"className":26073},[3262],[29,26075],{"className":26076,"style":3258},[171],[29,26078,23278],{"className":26079},[109],[29,26081,26083],{"className":26082},[109,3686],[29,26084,15311],{"className":26085},[109],[29,26087,154],{"className":26088},[153],[29,26090,26092],{"className":26091},[126],[29,26093,26095],{"className":26094,"style":17225},[130],[29,26096],{},[29,26098],{"className":26099},[490,1549],[10,26101,26102],{},"代进不同长度（可以直接运行验证）：",[22469,26104,26108],{"className":26105,"code":26106,"language":26107,"meta":22474,"style":22474},"language-python shiki shiki-themes github-light github-dark","P = 16e9        # activated 参数量\nW = 8e9         # FP4 权重，每 token 要读的字节数\nkv = 890        # 每 token 的 KV 字节数\ninflection = 1125  # B200 FP4 的算力带宽比（FLOP\u002Fbyte）\n\ndef intensity(B, N):\n    return B * 2 * P \u002F (W + B * kv * N)\n\nfor N in [4e3, 32e3, 128e3, 1e6]:\n    cells = \"  \".join(f\"B={B}:{intensity(B, N):>6.0f}\" for B in [1, 8, 64, 512])\n    mark = \"  \u003C- 越线\" if intensity(512, N) > inflection else \"\"\n    print(f\"N={N:>8.0f}  {cells}{mark}\")\n","python",[22476,26109,26110,26129,26143,26157,26171,26178,26191,26231,26236,26273,26344,26378],{"__ignoreMap":22474},[29,26111,26114,26118,26121,26125],{"class":26112,"line":26113},"line",1,[29,26115,26117],{"class":26116},"sVt8B","P ",[29,26119,508],{"class":26120},"szBVR",[29,26122,26124],{"class":26123},"sj4cs"," 16e9",[29,26126,26128],{"class":26127},"sJ8bj","        # activated 参数量\n",[29,26130,26132,26135,26137,26140],{"class":26112,"line":26131},2,[29,26133,26134],{"class":26116},"W ",[29,26136,508],{"class":26120},[29,26138,26139],{"class":26123}," 8e9",[29,26141,26142],{"class":26127},"         # FP4 权重，每 token 要读的字节数\n",[29,26144,26146,26149,26151,26154],{"class":26112,"line":26145},3,[29,26147,26148],{"class":26116},"kv ",[29,26150,508],{"class":26120},[29,26152,26153],{"class":26123}," 890",[29,26155,26156],{"class":26127},"        # 每 token 的 KV 字节数\n",[29,26158,26160,26163,26165,26168],{"class":26112,"line":26159},4,[29,26161,26162],{"class":26116},"inflection ",[29,26164,508],{"class":26120},[29,26166,26167],{"class":26123}," 1125",[29,26169,26170],{"class":26127},"  # B200 FP4 的算力带宽比（FLOP\u002Fbyte）\n",[29,26172,26174],{"class":26112,"line":26173},5,[29,26175,26177],{"emptyLinePlaceholder":26176},true,"\n",[29,26179,26181,26184,26188],{"class":26112,"line":26180},6,[29,26182,26183],{"class":26120},"def",[29,26185,26187],{"class":26186},"sScJk"," intensity",[29,26189,26190],{"class":26116},"(B, N):\n",[29,26192,26194,26197,26200,26203,26206,26209,26212,26214,26217,26219,26221,26223,26226,26228],{"class":26112,"line":26193},7,[29,26195,26196],{"class":26120},"    return",[29,26198,26199],{"class":26116}," B ",[29,26201,26202],{"class":26120},"*",[29,26204,26205],{"class":26123}," 2",[29,26207,26208],{"class":26120}," *",[29,26210,26211],{"class":26116}," P ",[29,26213,3083],{"class":26120},[29,26215,26216],{"class":26116}," (W ",[29,26218,3236],{"class":26120},[29,26220,26199],{"class":26116},[29,26222,26202],{"class":26120},[29,26224,26225],{"class":26116}," kv ",[29,26227,26202],{"class":26120},[29,26229,26230],{"class":26116}," N)\n",[29,26232,26234],{"class":26112,"line":26233},8,[29,26235,26177],{"emptyLinePlaceholder":26176},[29,26237,26239,26242,26245,26248,26251,26254,26257,26260,26262,26265,26267,26270],{"class":26112,"line":26238},9,[29,26240,26241],{"class":26120},"for",[29,26243,26244],{"class":26116}," N ",[29,26246,26247],{"class":26120},"in",[29,26249,26250],{"class":26116}," [",[29,26252,26253],{"class":26123},"4e3",[29,26255,26256],{"class":26116},", ",[29,26258,26259],{"class":26123},"32e3",[29,26261,26256],{"class":26116},[29,26263,26264],{"class":26123},"128e3",[29,26266,26256],{"class":26116},[29,26268,26269],{"class":26123},"1e6",[29,26271,26272],{"class":26116},"]:\n",[29,26274,26276,26279,26281,26285,26288,26291,26294,26297,26299,26302,26305,26307,26310,26313,26315,26318,26321,26323,26325,26327,26329,26331,26333,26335,26337,26339,26341],{"class":26112,"line":26275},10,[29,26277,26278],{"class":26116},"    cells ",[29,26280,508],{"class":26120},[29,26282,26284],{"class":26283},"sZZnC"," \"  \"",[29,26286,26287],{"class":26116},".join(",[29,26289,26290],{"class":26120},"f",[29,26292,26293],{"class":26283},"\"B=",[29,26295,26296],{"class":26123},"{",[29,26298,15311],{"class":26116},[29,26300,26301],{"class":26123},"}",[29,26303,26304],{"class":26283},":",[29,26306,26296],{"class":26123},[29,26308,26309],{"class":26116},"intensity(B, N)",[29,26311,26312],{"class":26120},":>6.0f",[29,26314,26301],{"class":26123},[29,26316,26317],{"class":26283},"\"",[29,26319,26320],{"class":26120}," for",[29,26322,26199],{"class":26116},[29,26324,26247],{"class":26120},[29,26326,26250],{"class":26116},[29,26328,57],{"class":26123},[29,26330,26256],{"class":26116},[29,26332,6035],{"class":26123},[29,26334,26256],{"class":26116},[29,26336,13611],{"class":26123},[29,26338,26256],{"class":26116},[29,26340,13709],{"class":26123},[29,26342,26343],{"class":26116},"])\n",[29,26345,26347,26350,26352,26355,26358,26361,26363,26366,26369,26372,26375],{"class":26112,"line":26346},11,[29,26348,26349],{"class":26116},"    mark ",[29,26351,508],{"class":26120},[29,26353,26354],{"class":26283}," \"  \u003C- 越线\"",[29,26356,26357],{"class":26120}," if",[29,26359,26360],{"class":26116}," intensity(",[29,26362,13709],{"class":26123},[29,26364,26365],{"class":26116},", N) ",[29,26367,26368],{"class":26120},">",[29,26370,26371],{"class":26116}," inflection ",[29,26373,26374],{"class":26120},"else",[29,26376,26377],{"class":26283}," \"\"\n",[29,26379,26381,26384,26386,26388,26391,26393,26395,26398,26400,26403,26406,26409,26412,26414,26416],{"class":26112,"line":26380},12,[29,26382,26383],{"class":26123},"    print",[29,26385,460],{"class":26116},[29,26387,26290],{"class":26120},[29,26389,26390],{"class":26283},"\"N=",[29,26392,26296],{"class":26123},[29,26394,3534],{"class":26116},[29,26396,26397],{"class":26120},":>8.0f",[29,26399,26301],{"class":26123},[29,26401,26402],{"class":26123},"  {",[29,26404,26405],{"class":26116},"cells",[29,26407,26408],{"class":26123},"}{",[29,26410,26411],{"class":26116},"mark",[29,26413,26301],{"class":26123},[29,26415,26317],{"class":26283},[29,26417,26418],{"class":26116},")\n",[10,26420,26421,26422,413,26475,26526,26527,26583,26584,26634],{},"输出里只有两格越线，都在 4k 那一行，而且要 ",[29,26423,26425,26444],{"className":26424},[32],[29,26426,26428],{"className":26427},[36],[38,26429,26430],{"xmlns":40},[42,26431,26432,26441],{},[45,26433,26434,26436,26439],{},[51,26435,15311],{},[59,26437,26438],{},"≥",[55,26440,13709],{},[89,26442,26443],{"encoding":91},"B \\ge 512",[29,26445,26447,26466],{"className":26446,"ariaHidden":61},[96],[29,26448,26450,26454,26457,26460,26463],{"className":26449},[100],[29,26451],{"className":26452,"style":26453},[104],"height:0.8193em;vertical-align:-0.136em;",[29,26455,15311],{"className":26456,"style":17886},[109,113],[29,26458],{"className":26459,"style":367},[171],[29,26461,26438],{"className":26462},[371],[29,26464],{"className":26465,"style":367},[171],[29,26467,26469,26472],{"className":26468},[100],[29,26470],{"className":26471,"style":3272},[104],[29,26473,13709],{"className":26474},[109],[29,26476,26478,26496],{"className":26477},[32],[29,26479,26481],{"className":26480},[36],[38,26482,26483],{"xmlns":40},[42,26484,26485,26493],{},[45,26486,26487,26489,26491],{},[51,26488,3534],{},[59,26490,8389],{},[55,26492,17058],{},[89,26494,26495],{"encoding":91},"N \\to 0",[29,26497,26499,26517],{"className":26498,"ariaHidden":61},[96],[29,26500,26502,26505,26508,26511,26514],{"className":26501},[100],[29,26503],{"className":26504,"style":3718},[104],[29,26506,3534],{"className":26507,"style":3667},[109,113],[29,26509],{"className":26510,"style":367},[171],[29,26512,8389],{"className":26513},[371],[29,26515],{"className":26516,"style":367},[171],[29,26518,26520,26523],{"className":26519},[100],[29,26521],{"className":26522,"style":3272},[104],[29,26524,17058],{"className":26525},[109]," 时 ",[29,26528,26530,26550],{"className":26529},[32],[29,26531,26533],{"className":26532},[36],[38,26534,26535],{"xmlns":40},[42,26536,26537,26547],{},[45,26538,26539,26541,26543,26545],{},[51,26540,16474],{},[59,26542,8389],{},[55,26544,14779],{},[51,26546,15311],{},[89,26548,26549],{"encoding":91},"I \\to 4B",[29,26551,26553,26571],{"className":26552,"ariaHidden":61},[96],[29,26554,26556,26559,26562,26565,26568],{"className":26555},[100],[29,26557],{"className":26558,"style":3718},[104],[29,26560,16474],{"className":26561,"style":16489},[109,113],[29,26563],{"className":26564,"style":367},[171],[29,26566,8389],{"className":26567},[371],[29,26569],{"className":26570,"style":367},[171],[29,26572,26574,26577,26580],{"className":26573},[100],[29,26575],{"className":26576,"style":3718},[104],[29,26578,14779],{"className":26579},[109],[29,26581,15311],{"className":26582,"style":17886},[109,113],"，所以 batch 要超过 280 才有可能碰到那条线。",[29,26585,26587,26604],{"className":26586},[32],[29,26588,26590],{"className":26589},[36],[38,26591,26592],{"xmlns":40},[42,26593,26594,26602],{},[45,26595,26596,26598,26600],{},[51,26597,15311],{},[59,26599,508],{},[55,26601,13611],{},[89,26603,18739],{"encoding":91},[29,26605,26607,26625],{"className":26606,"ariaHidden":61},[96],[29,26608,26610,26613,26616,26619,26622],{"className":26609},[100],[29,26611],{"className":26612,"style":3718},[104],[29,26614,15311],{"className":26615,"style":17886},[109,113],[29,26617],{"className":26618,"style":367},[171],[29,26620,508],{"className":26621},[371],[29,26623],{"className":26624,"style":367},[171],[29,26626,26628,26631],{"className":26627},[100],[29,26629],{"className":26630,"style":3272},[104],[29,26632,13611],{"className":26633},[109]," 的极限是 256，连门槛都摸不到。",[10,26636,26637,26665,26666,26526,26716,26797],{},[29,26638,26640,26653],{"className":26639},[32],[29,26641,26643],{"className":26642},[36],[38,26644,26645],{"xmlns":40},[42,26646,26647,26651],{},[45,26648,26649],{},[51,26650,3534],{},[89,26652,3534],{"encoding":91},[29,26654,26656],{"className":26655,"ariaHidden":61},[96],[29,26657,26659,26662],{"className":26658},[100],[29,26660],{"className":26661,"style":3718},[104],[29,26663,3534],{"className":26664,"style":3667},[109,113]," 一大就更没戏。",[29,26667,26669,26686],{"className":26668},[32],[29,26670,26672],{"className":26671},[36],[38,26673,26674],{"xmlns":40},[42,26675,26676,26684],{},[45,26677,26678,26680,26682],{},[51,26679,15311],{},[59,26681,8389],{},[51,26683,17294],{"mathvariant":1002},[89,26685,18845],{"encoding":91},[29,26687,26689,26707],{"className":26688,"ariaHidden":61},[96],[29,26690,26692,26695,26698,26701,26704],{"className":26691},[100],[29,26693],{"className":26694,"style":3718},[104],[29,26696,15311],{"className":26697,"style":17886},[109,113],[29,26699],{"className":26700,"style":367},[171],[29,26702,8389],{"className":26703},[371],[29,26705],{"className":26706,"style":367},[171],[29,26708,26710,26713],{"className":26709},[100],[29,26711],{"className":26712,"style":537},[104],[29,26714,17294],{"className":26715},[109],[29,26717,26719,26749],{"className":26718},[32],[29,26720,26722],{"className":26721},[36],[38,26723,26724],{"xmlns":40},[42,26725,26726,26746],{},[45,26727,26728,26730,26732,26734,26736,26738,26740,26742,26744],{},[51,26729,16474],{},[59,26731,8389],{},[55,26733,69],{},[51,26735,16524],{},[51,26737,3083],{"mathvariant":1002},[59,26739,460],{"stretchy":459},[55,26741,24945],{},[51,26743,3534],{},[59,26745,466],{"stretchy":459},[89,26747,26748],{"encoding":91},"I \\to 2P \u002F (890N)",[29,26750,26752,26770],{"className":26751,"ariaHidden":61},[96],[29,26753,26755,26758,26761,26764,26767],{"className":26754},[100],[29,26756],{"className":26757,"style":3718},[104],[29,26759,16474],{"className":26760,"style":16489},[109,113],[29,26762],{"className":26763,"style":367},[171],[29,26765,8389],{"className":26766},[371],[29,26768],{"className":26769,"style":367},[171],[29,26771,26773,26776,26779,26782,26785,26788,26791,26794],{"className":26772},[100],[29,26774],{"className":26775,"style":479},[104],[29,26777,69],{"className":26778},[109],[29,26780,16524],{"className":26781,"style":2350},[109,113],[29,26783,3083],{"className":26784},[109],[29,26786,460],{"className":26787},[483],[29,26789,24945],{"className":26790},[109],[29,26792,3534],{"className":26793,"style":3667},[109,113],[29,26795,466],{"className":26796},[490],"，1M context 下只有 34，离 1125 差 33 倍。因为 KV 太小，把权重完全摊掉之后剩下的那点 KV 依然喂不饱算力。",[10,26799,26800,26801,26851,26852,26880],{},"结论比 MLA 那次更极端。MLA 至少留了一个\"大 batch 加短 context 会翻到 compute bound\"的窗口，而且那个窗口落在 4k，恰好是它收益最小的区间。ds4.1f 的窗口更窄：要 ",[29,26802,26804,26821],{"className":26803},[32],[29,26805,26807],{"className":26806},[36],[38,26808,26809],{"xmlns":40},[42,26810,26811,26819],{},[45,26812,26813,26815,26817],{},[51,26814,15311],{},[59,26816,26438],{},[55,26818,13709],{},[89,26820,26443],{"encoding":91},[29,26822,26824,26842],{"className":26823,"ariaHidden":61},[96],[29,26825,26827,26830,26833,26836,26839],{"className":26826},[100],[29,26828],{"className":26829,"style":26453},[104],[29,26831,15311],{"className":26832,"style":17886},[109,113],[29,26834],{"className":26835,"style":367},[171],[29,26837,26438],{"className":26838},[371],[29,26840],{"className":26841,"style":367},[171],[29,26843,26845,26848],{"className":26844},[100],[29,26846],{"className":26847,"style":3272},[104],[29,26849,13709],{"className":26850},[109]," 且 ",[29,26853,26855,26868],{"className":26854},[32],[29,26856,26858],{"className":26857},[36],[38,26859,26860],{"xmlns":40},[42,26861,26862,26866],{},[45,26863,26864],{},[51,26865,3534],{},[89,26867,3534],{"encoding":91},[29,26869,26871],{"className":26870,"ariaHidden":61},[96],[29,26872,26874,26877],{"className":26873},[100],[29,26875],{"className":26876,"style":3718},[104],[29,26878,3534],{"className":26879,"style":3667},[109,113]," 压到 1.4 万以内，才算勉强碰到算力那一侧。",[10,26882,26883],{},"再算一遍吞吐。每步耗时取带宽和算力里较大的那个：",[14645,26885,26886,26929],{},[14648,26887,26888],{},[14651,26889,26890,26920,26923,26926],{},[14654,26891,26892],{},[29,26893,26895,26908],{"className":26894},[32],[29,26896,26898],{"className":26897},[36],[38,26899,26900],{"xmlns":40},[42,26901,26902,26906],{},[45,26903,26904],{},[51,26905,15311],{},[89,26907,15311],{"encoding":91},[29,26909,26911],{"className":26910,"ariaHidden":61},[96],[29,26912,26914,26917],{"className":26913},[100],[29,26915],{"className":26916,"style":3718},[104],[29,26918,15311],{"className":26919,"style":17886},[109,113],[14654,26921,26922],{},"1M：单序列 tok\u002Fs",[14654,26924,26925],{},"1M：总吞吐",[14654,26927,26928],{},"32k：总吞吐",[14663,26930,26931,26941,26954,26967,26980,26992],{},[14651,26932,26933,26935,26937,26939],{},[14668,26934,57],{},[14668,26936,25661],{},[14668,26938,25661],{},[14668,26940,25625],{},[14651,26942,26943,26945,26948,26951],{},[14668,26944,6035],{},[14668,26946,26947],{},"517",[14668,26949,26950],{},"4,138",[14668,26952,26953],{},"7,773",[14651,26955,26956,26958,26961,26964],{},[14668,26957,23292],{},[14668,26959,26960],{},"211",[14668,26962,26963],{},"6,761",[14668,26965,26966],{},"28,657",[14651,26968,26969,26971,26974,26977],{},[14668,26970,3840],{},[14668,26972,26973],{},"63",[14668,26975,26976],{},"8,034",[14668,26978,26979],{},"87,276",[14651,26981,26982,26984,26986,26989],{},[14668,26983,13709],{},[14668,26985,23278],{},[14668,26987,26988],{},"8,431",[14668,26990,26991],{},"178,617",[14651,26993,26994,27040,27043,27046],{},[14668,26995,26996],{},[29,26997,26999,27015],{"className":26998},[32],[29,27000,27002],{"className":27001},[36],[38,27003,27004],{"xmlns":40},[42,27005,27006,27012],{},[45,27007,27008,27010],{},[59,27009,8389],{},[51,27011,17294],{"mathvariant":1002},[89,27013,27014],{"encoding":91},"\\to \\infty",[29,27016,27018,27031],{"className":27017,"ariaHidden":61},[96],[29,27019,27021,27025,27028],{"className":27020},[100],[29,27022],{"className":27023,"style":27024},[104],"height:0.3669em;",[29,27026,8389],{"className":27027},[371],[29,27029],{"className":27030,"style":367},[171],[29,27032,27034,27037],{"className":27033},[100],[29,27035],{"className":27036,"style":537},[104],[29,27038,17294],{"className":27039},[109],[14668,27041,27042],{},"趋于 0",[14668,27044,27045],{},"8,572",[14668,27047,27048],{},"274,315",[10,27050,27051],{},[14,27052,27053],{},"总吞吐有上限，因为它最终被 KV 的带宽锁住：",[29,27055,27057],{"className":27056},[597],[29,27058,27060,27102],{"className":27059},[32],[29,27061,27063],{"className":27062},[36],[38,27064,27065],{"xmlns":40,"display":606},[42,27066,27067,27099],{},[45,27068,27069,27079,27081],{},[48,27070,27071,27074,27077],{},[76,27072,27073],{},"吞吐",[51,27075,27076],{},"max",[59,27078,1111],{},[59,27080,8389],{},[1088,27082,27083,27089],{},[45,27084,27085,27087],{},[55,27086,6035],{},[76,27088,15471],{},[45,27090,27091,27093,27095,27097],{},[55,27092,24945],{},[76,27094,25701],{},[59,27096,3514],{},[51,27098,3534],{},[89,27100,27101],{"encoding":91},"\\text{吞吐}_{\\max} \\to \\frac{8\\ \\text{TB\u002Fs}}{890\\ \\text{B} \\times N}",[29,27103,27105,27177],{"className":27104,"ariaHidden":61},[96],[29,27106,27108,27111,27168,27171,27174],{"className":27107},[100],[29,27109],{"className":27110,"style":4046},[104],[29,27112,27114,27120],{"className":27113},[109],[29,27115,27117],{"className":27116},[109,3686],[29,27118,27073],{"className":27119},[109,16407],[29,27121,27123],{"className":27122},[117],[29,27124,27126,27160],{"className":27125},[121,122],[29,27127,27129,27157],{"className":27128},[126],[29,27130,27132],{"className":27131,"style":3172},[130],[29,27133,27135,27138],{"style":27134},"top:-2.55em;margin-right:0.05em;",[29,27136],{"className":27137,"style":139},[138],[29,27139,27141],{"className":27140},[143,144,145,146],[29,27142,27144],{"className":27143},[109,146],[29,27145,27147,27151,27154],{"className":27146},[1576,146],[29,27148,27150],{"className":27149},[146],"m",[29,27152,6339],{"className":27153},[146],[29,27155,53],{"className":27156},[146],[29,27158,154],{"className":27159},[153],[29,27161,27163],{"className":27162},[126],[29,27164,27166],{"className":27165,"style":161},[130],[29,27167],{},[29,27169],{"className":27170,"style":367},[171],[29,27172,8389],{"className":27173},[371],[29,27175],{"className":27176,"style":367},[171],[29,27178,27180,27184],{"className":27179},[100],[29,27181],{"className":27182,"style":27183},[104],"height:2.1963em;vertical-align:-0.7693em;",[29,27185,27187,27190,27273],{"className":27186},[109],[29,27188],{"className":27189},[483,1549],[29,27191,27193],{"className":27192},[1088],[29,27194,27196,27265],{"className":27195},[121,122],[29,27197,27199,27262],{"className":27198},[126],[29,27200,27202,27234,27242],{"className":27201,"style":18032},[130],[29,27203,27204,27207],{"style":5413},[29,27205],{"className":27206,"style":5417},[138],[29,27208,27210,27213,27216,27222,27225,27228,27231],{"className":27209},[109],[29,27211,24945],{"className":27212},[109],[29,27214,5624],{"className":27215},[171],[29,27217,27219],{"className":27218},[109,3686],[29,27220,15311],{"className":27221},[109],[29,27223],{"className":27224,"style":3258},[171],[29,27226,3514],{"className":27227},[3262],[29,27229],{"className":27230,"style":3258},[171],[29,27232,3534],{"className":27233,"style":3667},[109,113],[29,27235,27236,27239],{"style":5426},[29,27237],{"className":27238,"style":5417},[138],[29,27240],{"className":27241,"style":1776},[1775],[29,27243,27244,27247],{"style":5435},[29,27245],{"className":27246,"style":5417},[138],[29,27248,27250,27253,27256],{"className":27249},[109],[29,27251,6035],{"className":27252},[109],[29,27254,5624],{"className":27255},[171],[29,27257,27259],{"className":27258},[109,3686],[29,27260,15532],{"className":27261},[109],[29,27263,154],{"className":27264},[153],[29,27266,27268],{"className":27267},[126],[29,27269,27271],{"className":27270,"style":17225},[130],[29,27272],{},[29,27274],{"className":27275},[490,1549],[14645,27277,27278,27320],{},[14648,27279,27280],{},[14651,27281,27282,27312,27314,27316,27318],{},[14654,27283,27284],{},[29,27285,27287,27300],{"className":27286},[32],[29,27288,27290],{"className":27289},[36],[38,27291,27292],{"xmlns":40},[42,27293,27294,27298],{},[45,27295,27296],{},[51,27297,3534],{},[89,27299,3534],{"encoding":91},[29,27301,27303],{"className":27302,"ariaHidden":61},[96],[29,27304,27306,27309],{"className":27305},[100],[29,27307],{"className":27308,"style":3718},[104],[29,27310,3534],{"className":27311,"style":3667},[109,113],[14654,27313,17788],{},[14654,27315,17805],{},[14654,27317,17821],{},[14654,27319,17837],{},[14663,27321,27322],{},[14651,27323,27324,27327,27330,27333,27336],{},[14668,27325,27326],{},"吞吐上限",[14668,27328,27329],{},"281k",[14668,27331,27332],{},"274k",[14668,27334,27335],{},"68.6k",[14668,27337,27338],{},"8.6k",[10,27340,27341,27342,27394],{},"4k 和 32k 那两档其实先被算力封顶（281k），再往上才是 KV 带宽封顶。分界在 ",[29,27343,27345,27364],{"className":27344},[32],[29,27346,27348],{"className":27347},[36],[38,27349,27350],{"xmlns":40},[42,27351,27352,27361],{},[45,27353,27354,27356,27358],{},[51,27355,3534],{},[59,27357,3843],{},[55,27359,27360],{},"3.2",[89,27362,27363],{"encoding":91},"N \\approx 3.2",[29,27365,27367,27385],{"className":27366,"ariaHidden":61},[96],[29,27368,27370,27373,27376,27379,27382],{"className":27369},[100],[29,27371],{"className":27372,"style":3718},[104],[29,27374,3534],{"className":27375,"style":3667},[109,113],[29,27377],{"className":27378,"style":367},[171],[29,27380,3843],{"className":27381},[371],[29,27383],{"className":27384,"style":367},[171],[29,27386,27388,27391],{"className":27387},[100],[29,27389],{"className":27390,"style":3272},[104],[29,27392,27360],{"className":27393},[109]," 万，和前面 batch 表里算出来的是同一个数。",[10,27396,27397,27400],{},[14,27398,27399],{},"代价在延迟那一列。"," 同一个 1M 的请求，batch 从 1 开到 512，单序列速度从 896 掉到 16 token\u002Fs。长 context 上吞吐和交互速度是直接对立的：要吃满吞吐，就得牺牲单个用户的等待时间。",[10,27402,27403],{},"反过来，如果要求单序列不低于 200 token\u002Fs，1M context 下 batch 只能开到 30 上下，单卡吞吐就是 6-7k token\u002Fs。这个量级和我实际跑 ds4.1f 的手感对得上。",[10,27405,27406],{},"换成 TP4 部署，每卡的账是这样：",[14645,27408,27409,27418],{},[14648,27410,27411],{},[14651,27412,27413,27415],{},[14654,27414],{},[14654,27416,27417],{},"每卡",[14663,27419,27420,27428,27436,27446,27456,27463],{},[14651,27421,27422,27425],{},[14668,27423,27424],{},"权重常驻显存（全部 552B 专家）",[14668,27426,27427],{},"69 GB",[14651,27429,27430,27433],{},[14668,27431,27432],{},"留给 KV 的显存",[14668,27434,27435],{},"111 GB",[14651,27437,27438,27441],{},[14668,27439,27440],{},"每步读的权重（16B activated 的四分之一）",[14668,27442,27443],{},[14,27444,27445],{},"2.00 GB",[14651,27447,27448,27451],{},[14668,27449,27450],{},"每步读的 KV（batch 1，1M context）",[14668,27452,27453],{},[14,27454,27455],{},"0.23 GB",[14651,27457,27458,27460],{},[14668,27459,25582],{},[14668,27461,27462],{},"2.23 GB",[14651,27464,27465,27468],{},[14668,27466,27467],{},"每步带宽下限",[14668,27469,27470],{},"0.28 ms",[10,27472,27473],{},"要分清常驻和每步读，也要交代那个除以 4 是哪来的。",[10,27475,27476,27479],{},[14,27477,27478],{},"常驻","：552B 个专家参数必须都在显存里。TP4 把每个权重矩阵切成四份、每卡持一份，所以是",[29,27481,27483],{"className":27482},[597],[29,27484,27486,27522],{"className":27485},[32],[29,27487,27489],{"className":27488},[36],[38,27490,27491],{"xmlns":40,"display":606},[42,27492,27493,27519],{},[45,27494,27495,27512,27514,27517],{},[1088,27496,27497,27510],{},[45,27498,27499,27502,27504,27506,27508],{},[55,27500,27501],{},"552",[76,27503,15311],{},[59,27505,3514],{},[55,27507,24844],{},[76,27509,16335],{},[55,27511,14779],{},[59,27513,508],{},[55,27515,27516],{},"69",[76,27518,19334],{},[89,27520,27521],{"encoding":91},"\\frac{552\\text{B} \\times 0.5\\ \\text{字节}}{4} = 69\\ \\text{GB}",[29,27523,27525,27629],{"className":27524,"ariaHidden":61},[96],[29,27526,27528,27531,27620,27623,27626],{"className":27527},[100],[29,27529],{"className":27530,"style":20815},[104],[29,27532,27534,27537,27617],{"className":27533},[109],[29,27535],{"className":27536},[483,1549],[29,27538,27540],{"className":27539},[1088],[29,27541,27543,27609],{"className":27542},[121,122],[29,27544,27546,27606],{"className":27545},[126],[29,27547,27549,27560,27568],{"className":27548,"style":15511},[130],[29,27550,27551,27554],{"style":5413},[29,27552],{"className":27553,"style":5417},[138],[29,27555,27557],{"className":27556},[109],[29,27558,14779],{"className":27559},[109],[29,27561,27562,27565],{"style":5426},[29,27563],{"className":27564,"style":5417},[138],[29,27566],{"className":27567,"style":1776},[1775],[29,27569,27570,27573],{"style":5435},[29,27571],{"className":27572,"style":5417},[138],[29,27574,27576,27579,27585,27588,27591,27594,27597,27600],{"className":27575},[109],[29,27577,27501],{"className":27578},[109],[29,27580,27582],{"className":27581},[109,3686],[29,27583,15311],{"className":27584},[109],[29,27586],{"className":27587,"style":3258},[171],[29,27589,3514],{"className":27590},[3262],[29,27592],{"className":27593,"style":3258},[171],[29,27595,24844],{"className":27596},[109],[29,27598,5624],{"className":27599},[171],[29,27601,27603],{"className":27602},[109,3686],[29,27604,16408],{"className":27605},[109,16407],[29,27607,154],{"className":27608},[153],[29,27610,27612],{"className":27611},[126],[29,27613,27615],{"className":27614,"style":5457},[130],[29,27616],{},[29,27618],{"className":27619},[490,1549],[29,27621],{"className":27622,"style":367},[171],[29,27624,508],{"className":27625},[371],[29,27627],{"className":27628,"style":367},[171],[29,27630,27632,27635,27638,27641],{"className":27631},[100],[29,27633],{"className":27634,"style":3718},[104],[29,27636,27516],{"className":27637},[109],[29,27639,5624],{"className":27640},[171],[29,27642,27644],{"className":27643},[109,3686],[29,27645,19430],{"className":27646},[109],[10,27648,27649,27652],{},[14,27650,27651],{},"每步读","：一个 token 只激活其中 16B。整机每步要读",[29,27654,27656],{"className":27655},[597],[29,27657,27659,27686],{"className":27658},[32],[29,27660,27662],{"className":27661},[36],[38,27663,27664],{"xmlns":40,"display":606},[42,27665,27666,27684],{},[45,27667,27668,27670,27672,27674,27676,27678,27680,27682],{},[55,27669,23278],{},[76,27671,15311],{},[59,27673,3514],{},[55,27675,24844],{},[76,27677,16335],{},[59,27679,508],{},[55,27681,6035],{},[76,27683,19334],{},[89,27685,24855],{"encoding":91},[29,27687,27689,27713,27740],{"className":27688,"ariaHidden":61},[96],[29,27690,27692,27695,27698,27704,27707,27710],{"className":27691},[100],[29,27693],{"className":27694,"style":3571},[104],[29,27696,23278],{"className":27697},[109],[29,27699,27701],{"className":27700},[109,3686],[29,27702,15311],{"className":27703},[109],[29,27705],{"className":27706,"style":3258},[171],[29,27708,3514],{"className":27709},[3262],[29,27711],{"className":27712,"style":3258},[171],[29,27714,27716,27719,27722,27725,27731,27734,27737],{"className":27715},[100],[29,27717],{"className":27718,"style":3718},[104],[29,27720,24844],{"className":27721},[109],[29,27723,5624],{"className":27724},[171],[29,27726,27728],{"className":27727},[109,3686],[29,27729,16408],{"className":27730},[109,16407],[29,27732],{"className":27733,"style":367},[171],[29,27735,508],{"className":27736},[371],[29,27738],{"className":27739,"style":367},[171],[29,27741,27743,27746,27749,27752],{"className":27742},[100],[29,27744],{"className":27745,"style":3718},[104],[29,27747,6035],{"className":27748},[109],[29,27750,5624],{"className":27751},[171],[29,27753,27755],{"className":27754},[109,3686],[29,27756,19430],{"className":27757},[109],[10,27759,27760],{},"而这 16B 激活参数同样散在四张卡上，各卡只读自己那一份，所以每卡读四分之一：",[29,27762,27764],{"className":27763},[597],[29,27765,27767,27795],{"className":27766},[32],[29,27768,27770],{"className":27769},[36],[38,27771,27772],{"xmlns":40,"display":606},[42,27773,27774,27792],{},[45,27775,27776,27786,27788,27790],{},[1088,27777,27778,27784],{},[45,27779,27780,27782],{},[55,27781,6035],{},[76,27783,19334],{},[55,27785,14779],{},[59,27787,508],{},[55,27789,69],{},[76,27791,19334],{},[89,27793,27794],{"encoding":91},"\\frac{8\\ \\text{GB}}{4} = 2\\ \\text{GB}",[29,27796,27798,27884],{"className":27797,"ariaHidden":61},[96],[29,27799,27801,27804,27875,27878,27881],{"className":27800},[100],[29,27802],{"className":27803,"style":20815},[104],[29,27805,27807,27810,27872],{"className":27806},[109],[29,27808],{"className":27809},[483,1549],[29,27811,27813],{"className":27812},[1088],[29,27814,27816,27864],{"className":27815},[121,122],[29,27817,27819,27861],{"className":27818},[126],[29,27820,27822,27833,27841],{"className":27821,"style":15511},[130],[29,27823,27824,27827],{"style":5413},[29,27825],{"className":27826,"style":5417},[138],[29,27828,27830],{"className":27829},[109],[29,27831,14779],{"className":27832},[109],[29,27834,27835,27838],{"style":5426},[29,27836],{"className":27837,"style":5417},[138],[29,27839],{"className":27840,"style":1776},[1775],[29,27842,27843,27846],{"style":5435},[29,27844],{"className":27845,"style":5417},[138],[29,27847,27849,27852,27855],{"className":27848},[109],[29,27850,6035],{"className":27851},[109],[29,27853,5624],{"className":27854},[171],[29,27856,27858],{"className":27857},[109,3686],[29,27859,19430],{"className":27860},[109],[29,27862,154],{"className":27863},[153],[29,27865,27867],{"className":27866},[126],[29,27868,27870],{"className":27869,"style":5457},[130],[29,27871],{},[29,27873],{"className":27874},[490,1549],[29,27876],{"className":27877,"style":367},[171],[29,27879,508],{"className":27880},[371],[29,27882],{"className":27883,"style":367},[171],[29,27885,27887,27890,27893,27896],{"className":27886},[100],[29,27888],{"className":27889,"style":3718},[104],[29,27891,69],{"className":27892},[109],[29,27894,5624],{"className":27895},[171],[29,27897,27899],{"className":27898},[109,3686],[29,27900,19430],{"className":27901},[109],[10,27903,27904],{},"KV 是同一个道理。TP 把 KV head 也分到四张卡上，每卡只存自己那部分，也就只读自己那部分：",[29,27906,27908],{"className":27907},[597],[29,27909,27911,27948],{"className":27910},[32],[29,27912,27914],{"className":27913},[36],[38,27915,27916],{"xmlns":40,"display":606},[42,27917,27918,27945],{},[45,27919,27920,27938,27940,27943],{},[1088,27921,27922,27936],{},[45,27923,27924,27926,27928,27930],{},[55,27925,24945],{},[76,27927,25701],{},[59,27929,3514],{},[301,27931,27932,27934],{},[55,27933,16930],{},[55,27935,24954],{},[55,27937,14779],{},[59,27939,508],{},[55,27941,27942],{},"0.23",[76,27944,19334],{},[89,27946,27947],{"encoding":91},"\\frac{890\\ \\text{B} \\times 10^6}{4} = 0.23\\ \\text{GB}",[29,27949,27951,28079],{"className":27950,"ariaHidden":61},[96],[29,27952,27954,27958,28070,28073,28076],{"className":27953},[100],[29,27955],{"className":27956,"style":27957},[104],"height:2.1771em;vertical-align:-0.686em;",[29,27959,27961,27964,28067],{"className":27960},[109],[29,27962],{"className":27963},[483,1549],[29,27965,27967],{"className":27966},[1088],[29,27968,27970,28059],{"className":27969},[121,122],[29,27971,27973,28056],{"className":27972},[126],[29,27974,27976,27987,27995],{"className":27975,"style":17028},[130],[29,27977,27978,27981],{"style":5413},[29,27979],{"className":27980,"style":5417},[138],[29,27982,27984],{"className":27983},[109],[29,27985,14779],{"className":27986},[109],[29,27988,27989,27992],{"style":5426},[29,27990],{"className":27991,"style":5417},[138],[29,27993],{"className":27994,"style":1776},[1775],[29,27996,27997,28000],{"style":5435},[29,27998],{"className":27999,"style":5417},[138],[29,28001,28003,28006,28009,28015,28018,28021,28024,28027],{"className":28002},[109],[29,28004,24945],{"className":28005},[109],[29,28007,5624],{"className":28008},[171],[29,28010,28012],{"className":28011},[109,3686],[29,28013,15311],{"className":28014},[109],[29,28016],{"className":28017,"style":3258},[171],[29,28019,3514],{"className":28020},[3262],[29,28022],{"className":28023,"style":3258},[171],[29,28025,57],{"className":28026},[109],[29,28028,28030,28033],{"className":28029},[109],[29,28031,17058],{"className":28032},[109],[29,28034,28036],{"className":28035},[117],[29,28037,28039],{"className":28038},[121],[29,28040,28042],{"className":28041},[126],[29,28043,28045],{"className":28044,"style":4490},[130],[29,28046,28047,28050],{"style":403},[29,28048],{"className":28049,"style":139},[138],[29,28051,28053],{"className":28052},[143,144,145,146],[29,28054,24954],{"className":28055},[109,146],[29,28057,154],{"className":28058},[153],[29,28060,28062],{"className":28061},[126],[29,28063,28065],{"className":28064,"style":5457},[130],[29,28066],{},[29,28068],{"className":28069},[490,1549],[29,28071],{"className":28072,"style":367},[171],[29,28074,508],{"className":28075},[371],[29,28077],{"className":28078,"style":367},[171],[29,28080,28082,28085,28088,28091],{"className":28081},[100],[29,28083],{"className":28084,"style":3718},[104],[29,28086,27942],{"className":28087},[109],[29,28089,5624],{"className":28090},[171],[29,28092,28094],{"className":28093},[109,3686],[29,28095,19430],{"className":28096},[109],[10,28098,28099,28102],{},[14,28100,28101],{},"两项加起来，KV 只占 10%。"," 和前面 batch 1 的结论一致，TP 不改变这个比例。",[10,28104,28105],{},"四张卡合起来，节点级 KV 容量是 444 GB，按 890 字节 \u002F token 算就是 4.99 亿 token。单序列的理论上限也从 896 提到约 3,600 token\u002Fs。",[10,28107,28108],{},"这就是 ds4.1f-FP4 在 B200 上的 roofline。",[10,28110,28111,28112,28115],{},"拐点 1125 FLOP\u002Fbyte 换一种读法：",[14,28113,28114],{},"搬运 1 个字节的时间，够做 1125 次浮点运算","。于是一个机制只要能用不到 1125 次运算省下 1 个字节的搬运，就净赚。以这个比值衡量，ds4.1f 里的几个设计都在做同一笔交易：",[15609,28117,28118,28201,28207],{},[15612,28119,28120,28123,28124,28200],{},[14,28121,28122],{},"SWA Bounded Replay","：请求恢复时不重放整条序列，只重算最近 ",[29,28125,28127,28145],{"className":28126},[32],[29,28128,28130],{"className":28129},[36],[38,28131,28132],{"xmlns":40},[42,28133,28134,28142],{},[45,28135,28136],{},[48,28137,28138,28140],{},[51,28139,1013],{},[76,28141,24314],{},[89,28143,28144],{"encoding":91},"n_{\\text{win}}",[29,28146,28148],{"className":28147,"ariaHidden":61},[96],[29,28149,28151,28154],{"className":28150},[100],[29,28152],{"className":28153,"style":692},[104],[29,28155,28157,28160],{"className":28156},[109],[29,28158,1013],{"className":28159},[109,113],[29,28161,28163],{"className":28162},[117],[29,28164,28166,28192],{"className":28165},[121,122],[29,28167,28169,28189],{"className":28168},[126],[29,28170,28172],{"className":28171,"style":4128},[130],[29,28173,28174,28177],{"style":134},[29,28175],{"className":28176,"style":139},[138],[29,28178,28180],{"className":28179},[143,144,145,146],[29,28181,28183],{"className":28182},[109,146],[29,28184,28186],{"className":28185},[109,3686,146],[29,28187,24314],{"className":28188},[109,146],[29,28190,154],{"className":28191},[153],[29,28193,28195],{"className":28194},[126],[29,28196,28198],{"className":28197,"style":161},[130],[29,28199],{}," 个 token 来重建局部状态。用一段计算，换掉局部窗口 KV 的持久化。",[15612,28202,28203,28206],{},[14,28204,28205],{},"FP4 KV","：存储和带宽都降到四分之一，代价是读进来要反量化。用一点计算，换掉四分之三的字节。",[15612,28208,28209,28212],{},[14,28210,28211],{},"Hierarchical Indexer","：先用一层打分粗筛出候选池，后续层只在这个池子里挑。用一层索引，换掉对全部候选的扫描。",[10,28214,28215],{},"共同点是把成本从 memory 一侧挪到 compute 一侧。",[10,28217,28218,28219],{},"前面几代的动作一直是压缩存储。现在存储已经压到接近下限，规则反过来：",[14,28220,28221],{},"用富余的算力，去重建那些不值得搬过来的字节。",[10,28223,28224],{},"这些省下来的资源最终可以被重新花到：更多 tokens、更好的数据过滤、更长的 RL rollout、更多 environment interaction、更多 rejection sampling、更多 verifier compute、更大的 batch、更复杂的 agent tasks。到这里，可以回到最初的问题了。ds4.1f 省掉的，是账单上几乎每一项。",[21,28226,28228],{"id":28227},"capability-与-efficiency","Capability 与 Efficiency",[10,28230,28231],{},"业界基本有共识：能力里很大的一部分来自后训练，而不是架构。三个例子。",[10,28233,28234,28235,28240],{},"GLM-5.3 的发布说明只有一句话，",[6339,28236,28239],{"href":28237,"rel":28238},"https:\u002F\u002Fz.ai\u002Fblog\u002Fglm-5.3",[6343],"Scaling post-training is all we did","。基座和 GLM-5.2 一样，没有新增一个预训练 token，Terminal-Bench 3.0 从 4.6 涨到 28.3，六倍。",[10,28242,28243,28244,28249],{},"Cognition 的 ",[6339,28245,28248],{"href":28246,"rel":28247},"https:\u002F\u002Fcognition.com\u002Fblog\u002Fswe-2",[6343],"SWE-2"," 更能说明分工。它是一个美国实验室从 Kimi K3 的开源权重上后训练出来的，而 Kimi K3 是 Moonshot 放出来的 2.8T 参数模型。Cognition 报的成绩是编码能力离前沿只差一点，成本低 64%；首次真正动手改代码的中位步数从 48 降到 18。",[10,28251,28252],{},"开源权重到了这个水平，做一款有竞争力的编码产品，贵的环节已经不是预训练，而是后训练和 harness。",[10,28254,28255],{},"第三个例子是 DeepSeek V4-Flash-0731。它的模型结构和尺寸与 Preview 保持一致，只重新做了后训练，但 Agent benchmark 大幅提升：",[10,28257,28258,28259,28262],{},"DeepSWE 从 7.3 涨到 54.4，CyberGym 从 38.7 到 76.7，TerminalBench 2.1 从 61.8 到 82.7。模型 backbone 没换。Attention 没换。参数量没换。官方更新日志直接写明结构不变、仅重新后训练。这个例子比任何口号都说明问题：",[14,28260,28261],{},"同一个 backbone 的 post-training 差异，完全可能比两代 architecture 的差异更大。"," 所以把\"模型进步\"拆成四个层次更合适：",[14645,28264,28265,28278],{},[14648,28266,28267],{},[14651,28268,28269,28272,28275],{},[14654,28270,28271],{},"层次",[14654,28273,28274],{},"解决什么",[14654,28276,28277],{},"对能力的作用",[14663,28279,28280,28291,28302,28313],{},[14651,28281,28282,28285,28288],{},[14668,28283,28284],{},"Architecture",[14668,28286,28287],{},"每 FLOP \u002F 每 GB 能做多少事",[14668,28289,28290],{},"提高上限和效率",[14651,28292,28293,28296,28299],{},[14668,28294,28295],{},"Data",[14668,28297,28298],{},"模型到底看到了什么分布",[14668,28300,28301],{},"决定知识、能力覆盖",[14651,28303,28304,28307,28310],{},[14668,28305,28306],{},"Post-training \u002F RL",[14668,28308,28309],{},"学会怎么推理、用工具、完成任务",[14668,28311,28312],{},"直接决定最终行为能力",[14651,28314,28315,28318,28321],{},[14668,28316,28317],{},"Infra",[14668,28319,28320],{},"上面三件事是否真的按设计运行",[14668,28322,28323],{},"防止几个月训练被 bug 偷走",[10,28325,28326,28327,28330,28331,28334],{},"Infra 这一层最容易被低估。RL 的产出取决于一条很长的系统，公式只占其中一环：",[22476,28328,28329],{},"prompt → rollout → tool environment → reward → sampling → advantage → gradient → model update","。任何地方一个小 bug，最后都会变成同一句话：\"奇怪，我们这个新的 RL 算法怎么效果一般？\" 而那个 bug 可能是 reward 算错、tool state 泄漏、rollout truncation 错、mask 错、sampling distribution 不一致、tokenizer offset 错、advantage normalization 错、stale policy 太严重，或者 sandbox reset 有问题。实际上不是算法一般，是代码在偷偷犯罪。所以更准的因果链是架构先降低成本，省下的成本换来更有效的训练，训练再沉淀成能力；\"架构直接产生智能\"是另一种叙事。架构对能力的作用经常是",[14,28332,28333],{},"间接的","。如果问\"这一代模型为什么明显更好用了\"，重要性排序大致是 Data > Post-training\u002FRL > Eval\u002FVerifier > Training infra > Architecture，答案多半落在前面几项，而不在\"我们把 attention 又改了一个 acronym\"。旁边还有一组更好记的说法，把每一层对应到一个\"前沿\"：",[15609,28336,28337,28344,28351,28358],{},[15612,28338,28339,28340,28343],{},"Architecture 决定 ",[14,28341,28342],{},"efficiency frontier","：每单位算力能处理多少有效训练信号",[15612,28345,28346,28347,28350],{},"Data 决定 ",[14,28348,28349],{},"learning frontier","：模型有机会学到什么",[15612,28352,28353,28354,28357],{},"RL \u002F post-training 决定 ",[14,28355,28356],{},"behavior frontier","：模型会不会把能力真正用出来",[15612,28359,28360,28361,28364],{},"Infra 决定",[14,28362,28363],{},"你是否真的获得前三者","：实验结果到底是不是你以为的实验",[10,28366,28367],{},"Infra 这一条听起来最不起眼，出问题时损失最大。一个漂亮的新 loss 写在论文里只需要半页；一个可靠的 million-token agent rollout system，可能需要几十个人跟各种分布式 bug、CUDA bug、环境 bug、数据 bug 打几个月架。论文不会把凌晨三点的 NCCL timeout 写进公式里，但模型能力很可能就死在那里。KDA 和 CSA\u002FHCA 的作用前面已经说过：让 1M context 和 long-horizon RL rollout 在计算上变得现实。放进 infra 的语境再看一层：这套系统跑不起来，再好的 loss 也只是半页论文。",[10,28369,28370],{},[14,28371,28372],{},"架构是 compute multiplier，乘出来的是训练预算，不是智能本身。",[21,28374,28375],{"id":28375},"算力预算",[10,28377,28378],{},"如果架构主要在省钱，那\"该不该冒架构的险\"就变成了一个纯粹的预算问题。假设已验证的 Transformer scaling recipe 能拿到 95 分。新架构理论上能到 100，但有 20% 概率训练炸掉，只剩 70。如果算力富裕，为什么要冒险？把已验证的架构再 scale 10 倍，可能直接拿到 98，而且更可靠。这其实是一种\"富人的保守\"。而算力紧张的实验室没有 10× compute 可以挥霍，只能想办法让 1× compute 做出 3× compute 的效果，架构创新的 ROI 因此变得非常高。从优化目标看更直白。算力富裕的一方更接近：",[29,28380,28382],{"className":28381},[597],[29,28383,28385,28411],{"className":28384},[32],[29,28386,28388],{"className":28387},[36],[38,28389,28390],{"xmlns":40,"display":606},[42,28391,28392,28408],{},[45,28393,28394,28396,28398,28401,28403,28405],{},[51,28395,27076],{},[59,28397,1111],{},[76,28399,28400],{},"Capability",[59,28402,62],{"separator":61},[171,28404],{"width":2114},[76,28406,28407],{},"s.t. Compute 很大",[89,28409,28410],{"encoding":91},"\\max \\text{Capability}, \\quad \\text{s.t. Compute 很大}",[29,28412,28414],{"className":28413,"ariaHidden":61},[96],[29,28415,28417,28420,28423,28426,28432,28435,28438,28441],{"className":28416},[100],[29,28418],{"className":28419,"style":556},[104],[29,28421,27076],{"className":28422},[1576],[29,28424],{"className":28425,"style":172},[171],[29,28427,28429],{"className":28428},[109,3686],[29,28430,28400],{"className":28431},[109],[29,28433,62],{"className":28434},[167],[29,28436],{"className":28437,"style":2506},[171],[29,28439],{"className":28440,"style":172},[171],[29,28442,28444,28448],{"className":28443},[109,3686],[29,28445,28447],{"className":28446},[109],"s.t. Compute ",[29,28449,28451],{"className":28450},[109,16407],"很大",[10,28453,28454],{},"所以可以接受较低的 architecture efficiency，只要 scaling 稳。而算力受限的一方更像：",[29,28456,28458],{"className":28457},[597],[29,28459,28461,28484],{"className":28460},[32],[29,28462,28464],{"className":28463},[36],[38,28465,28466],{"xmlns":40,"display":606},[42,28467,28468,28481],{},[45,28469,28470,28472,28474],{},[51,28471,27076],{},[59,28473,1111],{},[1088,28475,28476,28478],{},[76,28477,28400],{},[76,28479,28480],{},"Compute",[89,28482,28483],{"encoding":91},"\\max \\frac{\\text{Capability}}{\\text{Compute}}",[29,28485,28487],{"className":28486,"ariaHidden":61},[96],[29,28488,28490,28494,28497,28500],{"className":28489},[100],[29,28491],{"className":28492,"style":28493},[104],"height:2.2519em;vertical-align:-0.8804em;",[29,28495,27076],{"className":28496},[1576],[29,28498],{"className":28499,"style":172},[171],[29,28501,28503,28506,28567],{"className":28502},[109],[29,28504],{"className":28505},[483,1549],[29,28507,28509],{"className":28508},[1088],[29,28510,28512,28558],{"className":28511},[121,122],[29,28513,28515,28555],{"className":28514},[126],[29,28516,28519,28533,28541],{"className":28517,"style":28518},[130],"height:1.3714em;",[29,28520,28521,28524],{"style":5413},[29,28522],{"className":28523,"style":5417},[138],[29,28525,28527],{"className":28526},[109],[29,28528,28530],{"className":28529},[109,3686],[29,28531,28480],{"className":28532},[109],[29,28534,28535,28538],{"style":5426},[29,28536],{"className":28537,"style":5417},[138],[29,28539],{"className":28540,"style":1776},[1775],[29,28542,28543,28546],{"style":5435},[29,28544],{"className":28545,"style":5417},[138],[29,28547,28549],{"className":28548},[109],[29,28550,28552],{"className":28551},[109,3686],[29,28553,28400],{"className":28554},[109],[29,28556,154],{"className":28557},[153],[29,28559,28561],{"className":28560},[126],[29,28562,28565],{"className":28563,"style":28564},[130],"height:0.8804em;",[29,28566],{},[29,28568],{"className":28569},[490,1549],[10,28571,28572],{},"于是被逼出 MLA、MoE、Sparse、Linear、各种 compression。这和芯片行业很像。资源充裕时加晶体管，资源受限时改 architecture。所以很多所谓架构路线之争，底层其实是不同算力预算下的经济学。把这件事再拆细一点，会发现竞争其实发生在四种资本上：",[15609,28574,28575,28581,28587,28593],{},[15612,28576,28577,28580],{},[14,28578,28579],{},"Compute Capital","：GPU、HBM、network、datacenter。",[15612,28582,28583,28586],{},[14,28584,28585],{},"Data Capital","：真实用户、agent trajectories、enterprise workflow、feedback。",[15612,28588,28589,28592],{},[14,28590,28591],{},"Algorithmic Capital","：architecture、training recipe、RL、verifier。",[15612,28594,28595,28598],{},[14,28596,28597],{},"Infrastructure Capital","：稳定地把几千几万张 GPU、RL environment、rollout 全跑起来。",[10,28600,28601,28602],{},"算力富裕的一方在前两项和第四项上有巨大积累；算力受限的一方被迫在第三项特别凶。这也是为什么\"美国靠暴力 scaling、中国靠聪明架构\"这种说法太简化了。更接近事实的写法是：",[14,28603,28604],{},"一边在用资源优势降低研究风险，另一边在用资源约束提高创新收益。",[10,28606,28607],{},"而且\"数据\"这一项也没法简单说成谁多谁少。真正稀缺的早就不是互联网文本了。Wikipedia、Common Crawl、GitHub 这些 public corpus，大家拿到的量级差不多。现在值钱的是高质量 reasoning traces、真实 agent trajectories、用户交互反馈、专业任务数据、tool-use traces、可验证环境。这些东西不是爬虫能给你的，是产品规模给的。",[10,28609,28610],{},"所以更要命的不是\"谁有更多互联网 token\"，而是\"谁有更多高价值的 interaction token\"。但这里有个反转。",[10,28612,28613],{},"一旦某个架构优化被证明有效，情况就变了。DeepSeek 做出 MLA，或者 Kimi 做出成熟的 Linear Attention，算力多的一方并不会因为自己算力多就拒绝使用。它们完全可以拿\"更多算力 × 更高效架构\"叠加使用。于是原本弱者为了生存开发出来的 efficiency innovation，最后反而可能让强者 scale 得更狠。问题最后又回到：省下来的钱，值不值得花。",[21,28615,28616],{"id":28616},"小结",[10,28618,28619],{},"开头说 ds4.1f 是模型架构与硬件 co-design 的杰作。看完整条路线，我觉得这个说法可以讲得更具体一点：它省掉的不是某一项。是账单上的好几项，而且是分几代攒下来的。把整条路线画成一张图是这样：",[22469,28621,28624],{"className":28622,"code":28623,"language":3686,"meta":22474},[22472],"MHA              每个 head 一份 K、V\n │\n ▼\nMQA \u002F GQA        压 head 数：离散地共享\n │\n ▼\nMLA              压 D 轴：连续压缩成 latent\n │\n ▼\nCSA \u002F HCA        压 N 轴：压成 block，再挑着看\n │\n ▼\nLinear \u002F SSM     压掉保存本身：只留一个 state\n │\n ▼\nV4.1             压 L 轴和计算阶段：不再每层每步都付同样的钱\n",[22476,28625,28623],{"__ignoreMap":22474},[10,28627,28628,28629,28632],{},"同样的路线可以读成一连串追问：每个 head 都需要独立 KV 吗——MQA\u002FGQA 说不用；每个 token 都需要完整 KV 吗——MLA 说不用；每个 query 都需要看所有 token 吗——CSA\u002FHCA 说不用；每个 token 都需要被永久保存吗——Linear\u002FSSM 说不用；每个 layer 都需要独立 memory、prefill 和 decode 必须用同样计算吗——V4.1 说不用。每一代都在放松一个假设，",[14,28630,28631],{},"保存信息、压缩信息、找回信息","三个动作在其中循环。",[10,28634,28635],{},"前面一直把架构、数据、算力当成三样不同的东西比轻重，这个切法可能一开始就偏了。它们互相兑换：",[10,28637,28638,28641],{},[14,28639,28640],{},"数据是架构。"," 用什么数据、怎么组织、怎么采样，直接决定模型长成什么样。同一套 attention，换一个数据分布就是另一个模型。数据管线里每个决定，效果上和改架构是同一个量级。",[10,28643,28644,28647],{},[14,28645,28646],{},"架构是算力。"," 这条前面已经推过一遍。MLA 把 KV 压到 1\u002F57，ds4.1f 把 decode 的算术强度压到离拐点 300 倍，省下来的都是算力。架构不改模型会做什么，只改做同一件事要花多少钱。它是算力的另一种记账方式。",[10,28649,28650,28653],{},[14,28651,28652],{},"算力是数据。"," 省下来的算力不会消失，会被重新花到更多 tokens、更长的 rollout、更多 verifier compute。算力的下游就是数据。",[10,28655,28656],{},"三个词绕成一个环。能力和架构谁更重要，这个问题问错了对象：三者是同一笔资源的三种形态，换名字不改变总量，变的只是兑换率。成本压到足够低之后，架构这条线上剩下的竞争，只是决定那笔资源能兑换多少次。",[28658,28659,28660],"style",{},"html pre.shiki code .sVt8B, html code.shiki .sVt8B{--shiki-default:#24292E;--shiki-dark:#E1E4E8}html pre.shiki code .szBVR, html code.shiki .szBVR{--shiki-default:#D73A49;--shiki-dark:#F97583}html pre.shiki code .sj4cs, html code.shiki .sj4cs{--shiki-default:#005CC5;--shiki-dark:#79B8FF}html pre.shiki code .sJ8bj, html code.shiki .sJ8bj{--shiki-default:#6A737D;--shiki-dark:#6A737D}html pre.shiki code .sScJk, html code.shiki .sScJk{--shiki-default:#6F42C1;--shiki-dark:#B392F0}html pre.shiki code .sZZnC, html code.shiki .sZZnC{--shiki-default:#032F62;--shiki-dark:#9ECBFF}html .default .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .shiki span {color: var(--shiki-default);background: var(--shiki-default-bg);font-style: var(--shiki-default-font-style);font-weight: var(--shiki-default-font-weight);text-decoration: var(--shiki-default-text-decoration);}html .dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}html.dark .shiki span {color: var(--shiki-dark);background: var(--shiki-dark-bg);font-style: var(--shiki-dark-font-style);font-weight: var(--shiki-dark-font-weight);text-decoration: var(--shiki-dark-text-decoration);}",{"title":22474,"searchDepth":26131,"depth":26145,"links":28662},[28663,28664,28665,28666,28667,28671,28672,28675,28676,28677,28678],{"id":23,"depth":26131,"text":24},{"id":3951,"depth":26131,"text":3952},{"id":5024,"depth":26131,"text":5025},{"id":6443,"depth":26131,"text":6444},{"id":20078,"depth":26131,"text":20079,"children":28668},[28669,28670],{"id":20083,"depth":26145,"text":20084},{"id":20897,"depth":26145,"text":20898},{"id":21146,"depth":26131,"text":21147},{"id":22453,"depth":26131,"text":22454,"children":28673},[28674],{"id":22463,"depth":26145,"text":22464},{"id":22660,"depth":26131,"text":22661},{"id":28227,"depth":26131,"text":28228},{"id":28375,"depth":26131,"text":28375},{"id":28616,"depth":26131,"text":28616},"技术杂谈","2026-09-11 12:00:00","ds4.1f 上线了。它的架构 novel 在哪，便宜和快又是从哪省出来的。",false,"md",{},"\u002Fposts\u002Fllm-memory-compute-tradeoff",{"title":5,"description":28681},"posts\u002Fllm-memory-compute-tradeoff",[28689,1309,6444,28690,28691],"LLMs","DeepSeek","长上下文","OKZftJHCaSQqXoKyVZQ46CEMXa5JU3B-6v1j51qTRFI",[28694,28703,28709,28716,28721,28731,28740,28746,28750,28758,28767,28773,28779,28787,28791,28795,28797,28804,28810,28814,28818,28826,28833,28841,28846,28852,28858,28863,28868,28875,28882,28889,28895,28902,28910,28916,28922,28928,28934,28939,28948,28956],{"path":28695,"title":28696,"description":28697,"date":28698,"categories":28699,"tags":28700,"draft":26176},"\u002Fposts\u002Ftransposition-borrowed-chords","平移借用：大调里立刻能用的和弦","把大调整体平移，借用各调的顺阶和弦；有符号平移量 ±2、±3、±5、±7 给出七个近关系调，合计 25 个现成三和弦与 31 个现成七和弦，全部 12 个平移量给 36 与 48。","2026-11-08 16:20:00","音乐",[28701,28702],"乐理","和弦",{"path":28704,"title":28705,"description":28706,"date":28707,"categories":28699,"tags":28708,"draft":26176},"\u002Fposts\u002Fdiatonic-chords-major-system","大调体系下的三和弦和七和弦","以同主音 C 为基准，逐个推出七套音阶的顺阶三和弦与七和弦，标注每个和弦的出处，去重后得到 21 个三和弦与 28 个七和弦。","2026-11-08 15:20:00",[28701,28702],{"path":28710,"title":28711,"description":22474,"date":28712,"categories":28713,"tags":28714,"draft":26176},"\u002Fposts\u002F作息感慨","作息感慨","2026-11-04 07:15:08","生活杂谈",[28715],"日常",{"path":28717,"title":28718,"description":22474,"date":28719,"categories":22474,"tags":28720,"draft":26176},"\u002Fposts\u002Fgp-tab-测试","gp-tab-测试","2026-10-12 10:51:22",null,{"path":28722,"title":28723,"description":28724,"date":28725,"categories":28726,"tags":28727,"draft":28682},"\u002Fposts\u002Fnetwork-clean","怎么让网络变得干净","一个干净的网络，让人省心的使用服务。","2026-10-11 19:40:02","技术",[28728,28729,28730],"VPN","网络","AI",{"path":28732,"title":28733,"description":28734,"date":28735,"categories":28713,"tags":28736,"draft":28682},"\u002Fposts\u002Fthe-age-of-hesitation","[搬运] 不敢投入的时代","为什么我们现在越来越不敢投入了。很多时候我们以为的选择，比如你有两个工作offer，现在选择好像是我到底应该考公还是考编。明明这两个选项都不是真正的选项，因为你还没有开始，也不知道最后能不能成。这个时候我们就已经开始犹豫了。","2026-10-10 11:44:12",[28737,28738,28739],"翻译","搬运","心理学",{"path":28741,"title":28742,"description":28743,"date":28744,"categories":28713,"tags":28745,"draft":26176},"\u002Fposts\u002Fwhy-happiness-look-like-same","[搬运] 为什么大家的幸福都长一个样？","为什么大家的幸福都长一个样？一种认知被广泛传播后似乎变得不断扁平，这种感觉很奇怪。","2026-10-09 13:57:15",[28737,28738,28739],{"path":28747,"title":28748,"description":22474,"date":28749,"categories":28713,"tags":28720,"draft":26176},"\u002Fposts\u002Feverything-is-paid-twice","任何事情都要被消费两次","2026-10-06 12:30:06",{"path":28751,"title":28752,"description":28753,"date":28754,"categories":28679,"tags":28755,"draft":28682},"\u002Fposts\u002Fllm","为什么模型上线后，体感像是“降智”了？","权重与 KV cache 的量化都能降低 serving 成本，也可能改变输出；但体感变弱不足以证明服务后来切换了精度。","2026-09-29 21:44:08",[28689,28756,28757],"Serving","量化",{"path":28759,"title":28760,"description":28761,"date":28762,"categories":28726,"tags":28763,"draft":28682},"\u002Fposts\u002Fspinning-up-policy-optimization","[译] 深度强化学习入门（三）：策略优化入门","OpenAI Spinning Up 教程第三部分的全文翻译：推导最简单的策略梯度、log-derivative trick、EGLP 引理、reward-to-go 与基线，配 PyTorch 示例代码。","2026-09-28 17:49:25",[28764,28765,28766,28737],"强化学习","RL","Spinning Up",{"path":28768,"title":28769,"description":28770,"date":28771,"categories":28726,"tags":28772,"draft":28682},"\u002Fposts\u002Fspinning-up-rl-intro-2","[译] 深度强化学习入门（二）：强化学习算法的分类","OpenAI Spinning Up 第二部分的全文翻译：无模型与基于模型的 RL、策略优化与 Q-learning 的取舍，以及规划、专家迭代和经验增强等方法。","2026-09-28 10:20:21",[28764,28765,28766,28737],{"path":28774,"title":28775,"description":28776,"date":28777,"categories":28726,"tags":28778,"draft":28682},"\u002Fposts\u002Fspinning-up-rl-intro","[译] 深度强化学习入门（一）：RL 的关键概念","OpenAI Spinning Up 教程第一部分的全文翻译：智能体与环境、状态与观测、动作空间、策略、轨迹、奖励与回报、价值函数、贝尔曼方程、优势函数，附 MDP 形式化。","2026-09-28 09:29:40",[28764,28765,28766,28737],{"path":28780,"title":28781,"description":28782,"date":28783,"categories":28713,"tags":28784,"draft":26176},"\u002Fposts\u002Fwhen-machines-come-alive","冰箱之后，生活还会怎样改变？","从厨房里的果蝇想到冰箱、移动互联网与 AI 的多轮调用：AGI 已经到来，只是昂贵的智能会怎样改变生活？","2026-09-25 06:36:22",[28730,28785,28786],"随笔","科技",{"path":28788,"title":28789,"description":22474,"date":28790,"categories":22474,"tags":28720,"draft":26176},"\u002Fposts\u002Fmathematical-foundations-of-llms-1","LLM的数学基础（1）：","2026-09-19 22:59:55",{"path":28792,"title":28793,"description":22474,"date":28794,"categories":22474,"tags":28720,"draft":26176},"\u002Fposts\u002Ftowards-maximize-serving-capatity","如何在 offline bench 里最大化 LLM serving system 的吞吐","2026-09-19 05:45:05",{"path":28685,"title":5,"description":28681,"date":28680,"categories":28679,"tags":28796,"draft":28682},[28689,1309,6444,28690,28691],{"path":28798,"title":28799,"description":28800,"date":28801,"categories":28679,"tags":28802,"draft":26176},"\u002Fposts\u002Fmarkdown-extensions","Markdown 扩展语法速查","本站 Markdown 支持的扩展语法演示与速查：admonition 提示块、GitHub 仓库卡片、spoiler 遮罩、ABC 乐谱、GP 乐谱、音频播放器、浏览器可运行代码块。","2026-09-01 15:30:00",[28803],"博客",{"path":28805,"title":28806,"description":28807,"date":28808,"categories":28713,"tags":28809,"draft":26176},"\u002Fposts","为什么还需要手写博客？","AI这么强，还要古法写博客吗？","2026-09-01 13:17:53",[28803],{"path":28811,"title":28812,"description":22474,"date":28813,"categories":22474,"tags":28720,"draft":26176},"\u002Fposts\u002Fllm-serving-1","LLM Serving（一）:以在H200上部署GLM-5.2为例","2026-07-21 06:15:44",{"path":28815,"title":28816,"description":22474,"date":28817,"categories":28699,"tags":28720,"draft":28682},"\u002Fposts\u002Finteresting-music-rhythm","有趣的音乐节奏","2026-07-16 12:47:24",{"path":28819,"title":28820,"description":28821,"date":28822,"categories":28726,"tags":28823,"draft":28682},"\u002Fposts\u002Ftowards-monosemanticity","LLM 可解释性（一）：SAE与单语义","读 Anthropic《Towards Monosemanticity》。从难以解释的神经元出发, 讲清superposition为什么让单个神经元混成一团, SAE怎么把多语义的激活分解成单语义特征。","2026-07-12 22:00:00",[28689,28824,28825],"可解释性","SAE",{"path":28827,"title":28828,"description":28829,"date":28830,"categories":28713,"tags":28831,"draft":26176},"\u002Fposts\u002Fpiketty-ai-china-k-shaped","用皮凯蒂的镜头看 AI 与中国 K 型社会","一次凌晨的对话整理。从 r>g 出发，看 AI 时代的资本分配、中国的世袭化趋势、体制内的二元劳动力市场，以及一个普通 00 后在一线打拼的真实牌面。","2026-06-24 03:00:00",[28785,28832,28730],"经济",{"path":28834,"title":28835,"description":22474,"date":28836,"categories":22474,"tags":28837,"draft":26176},"\u002Fposts\u002Fclaude-code-compact-behavior","cluade code compact 行为","2026-06-23 20:17:42",[28838,28839,28840],"Agent","AI Agent","Claude Code",{"path":28842,"title":28843,"description":22474,"date":28844,"categories":28726,"tags":28845,"draft":26176},"\u002Fposts\u002Fclaude-code-settings","Claude Code 配置分享","2026-06-23 14:22:12",[28689,28839,28838,28840],{"path":28847,"title":28848,"description":28849,"date":28850,"categories":28679,"tags":28851,"draft":28682},"\u002Fposts\u002Fmy-thoughts-on-the-ai-industry","简单聊聊对 AI Agent 行业的理解","写这篇文章的初衷，是我发觉逐渐忘了前两个月的 AI\u002FLLMs 是什么样的能力和情况了。意识中似乎某些在突然加速，但理性和直觉告诉我这不对，作为行内人，我知道这是一个渐进的、累积的过程。我想自己回忆一下，Agent的发展。","2026-06-20 13:14:25",[28689,28839,28838],{"path":28853,"title":28854,"description":22474,"date":28855,"categories":28726,"tags":28856,"draft":28682},"\u002Fposts\u002Frotational-positional-encoding","RoPE","2026-06-17 18:31:40",[28689,28857,28854],"Transformer",{"path":28859,"title":28860,"description":22474,"date":28861,"categories":28726,"tags":28862,"draft":28682},"\u002Fposts\u002Fintroduction-to-large-language-model","大语言模型入门基础","2026-06-16 22:36:11",[28857,28689],{"path":28864,"title":28865,"description":22474,"date":28866,"categories":28679,"tags":28867,"draft":26176},"\u002Fposts\u002Fcoding-agent","关于 Coding Agent 使用的遐思","2026-06-03 17:54:20",[28838],{"path":28869,"title":28870,"description":28871,"date":28872,"categories":28726,"tags":28873,"draft":26176},"\u002Fposts\u002Fabout-ssh-usage","关于SSH的使用——客户端篇","介绍SSH客户端的用法。","2025-06-17 02:44:57",[28874],"工具",{"path":28876,"title":28877,"description":28878,"date":28879,"categories":28679,"tags":28880,"draft":28682},"\u002Fposts\u002Fopensourcesoftwarestarttutorial","如何开始一个开源项目？","浅谈我对开源项目的认知。","2025-04-17 18:00:00",[28881],"开源",{"path":28883,"title":28884,"description":28885,"date":28886,"categories":28887,"tags":28888,"draft":26176},"\u002Fposts\u002Fmotivation","渴求动力","有意识的生活并主动地行动会使得生活更美好。","2025-03-05 00:44:57","生活-情感",[28785],{"path":28890,"title":28891,"description":28892,"date":28893,"categories":28887,"tags":28894,"draft":26176},"\u002Fposts\u002F2024summary","2024年终总结","啊，痛彻的时光；啊，悲怆的人！","2025-01-01 01:00:00",[28785],{"path":28896,"title":28897,"description":28898,"date":28899,"categories":28726,"tags":28900,"draft":26176},"\u002Fposts\u002Fvps-crash-startup","快速搭建个人VPS","如何快速搭建个人VPS。","2024-12-31 19:00:00",[28901],"VPS",{"path":28903,"title":28904,"description":28905,"date":28906,"categories":28726,"tags":28907,"draft":26176},"\u002Fposts\u002Fdata-label-software","AI数据（自动）标注软件概述","对市面上目前的AI数据（自动）标注软件进行调研。","2024-12-20 14:00:00",[28908,28909],"数据标注","深度学习",{"path":28911,"title":28912,"description":28913,"date":28914,"categories":28887,"tags":28915,"draft":28682},"\u002Fposts\u002Fself-destruction","生活的崩坏与自我毁灭","是什么在牵引着我？我的行为，我的思想？我的感受？我所处的地方？它们在何处？","2024-12-19 12:00:00",[28785],{"path":28917,"title":28918,"description":28919,"date":28920,"categories":28726,"tags":28921,"draft":26176},"\u002Fposts\u002Fcs194-196_llmagents","LLM Agents 课程笔记","UCB CS294\u002F194-196 Large Language Model Agents 的课程笔记。","2024-11-30 20:00:00",[28838],{"path":28923,"title":28924,"description":28925,"date":28926,"categories":28887,"tags":28927,"draft":26176},"\u002Fposts\u002Fwhatsyourworth","价值、位置","你们的生存究竟有何价值？如果毫无价值，你们究竟为何存在？","2024-11-29 02:00:00",[28785],{"path":28929,"title":28930,"description":28931,"date":28932,"categories":28887,"tags":28933,"draft":28682},"\u002Fposts\u002Fblogrelifever2","重启博客v2","忙里偷闲学习前端，还是决定结合想法手搓一个自己的博客。","2024-11-25 00:00:00",[28803],{"path":28935,"title":28936,"description":28937,"date":28938,"categories":28887,"tags":28720,"draft":26176},"\u002Fposts\u002Fblogrelife","重启博客","在我的表达欲与现实所需(问题解决记录)驱动下，我决定重写博客。","2024-10-21 20:44:57",{"path":28940,"title":28941,"description":28942,"date":28943,"categories":28713,"tags":28944,"draft":28682},"\u002Fposts\u002Fsurvival-welcome-to-university","生存手册——欢迎来到大学","大学同时生产三样东西：可验证技能、认证信号和驯化资本。你付了四年的钱，真正拿到手的是哪一样，取决于你毕业时要进的那个市场在为什么付费。","2024-10-12 20:00:00",[28945,28946,28947],"大学","求职","方法论",{"path":28949,"title":28950,"description":28951,"date":28952,"categories":28887,"tags":28953,"draft":28682},"\u002Fposts\u002Frevival","240205 复活","好吧，断更了许久，我又终于回来了，我又终于回来了。","2024-02-05 20:00:00",[28785,28954,28955],"生活","返乡",{"path":28957,"title":28958,"description":28959,"date":28960,"categories":28887,"tags":28961,"draft":26176},"\u002Fposts\u002Fclouds","「230717」云","我已经很久忘记了不我已经很少把自己的想法在lofter上发布的。","2023-07-17 20:00:00",[28785,28962],"散文",1795115925453]