Implement structural updates and optimizations across multiple modules

src/xpu/flamegraph/profiler/src/main.rs

@@ -1,5 +1,7 @@
 use std::mem::MaybeUninit;
 use std::time::Duration;
+use std::sync::mpsc;
+use std::thread;
 
 use clap::ArgAction;
 use clap::Parser;
@@ -155,18 +157,31 @@ fn main() -> Result<(), libbpf_rs::Error> {
         event::OutputFormat::Standard
     };
 
+    let (tx, rx) = mpsc::channel::<Vec<u8>>();
+
+    // Spawn processing thread
+    let processing_thread = thread::spawn(move || {
         let event_handler = event::EventHandler::new(output_format);
+        for event_data in rx {
+            event_handler.handle(&event_data);
+        }
+    });
 
     let mut builder = libbpf_rs::RingBufferBuilder::new();
     builder
         .add(&skel.maps.events, move |data| {
-            event_handler.handle(data)
+            let _ = tx.send(data.to_vec());
+            0
         })
         .unwrap();
 
     let ringbuf = builder.build().unwrap();
     while ringbuf.poll(Duration::MAX).is_ok() {}
 
+    // Wait for processing thread to finish
+    drop(ringbuf);
+    let _ = processing_thread.join();
+
     // Clean up perf events if in perf mode
     if !pefds.is_empty() {
         perf::close_perf_events(pefds)?;

src/xpu/flamegraph/qwen3.cu/tensor_info.txt

@@ -0,0 +1,311 @@
+tensor[0]: name = output.weight, size = 622329856, offset = 0
+tensor[1]: name = output_norm.weight, size = 4096, offset = 622329856
+tensor[2]: name = token_embd.weight, size = 622329856, offset = 622333952
+tensor[3]: name = blk.0.attn_k.weight, size = 4194304, offset = 1244663808
+tensor[4]: name = blk.0.attn_k_norm.weight, size = 512, offset = 1248858112
+tensor[5]: name = blk.0.attn_norm.weight, size = 4096, offset = 1248858624
+tensor[6]: name = blk.0.attn_output.weight, size = 8388608, offset = 1248862720
+tensor[7]: name = blk.0.attn_q.weight, size = 8388608, offset = 1257251328
+tensor[8]: name = blk.0.attn_q_norm.weight, size = 512, offset = 1265639936
+tensor[9]: name = blk.0.attn_v.weight, size = 4194304, offset = 1265640448
+tensor[10]: name = blk.0.ffn_down.weight, size = 12582912, offset = 1269834752
+tensor[11]: name = blk.0.ffn_gate.weight, size = 12582912, offset = 1282417664
+tensor[12]: name = blk.0.ffn_norm.weight, size = 4096, offset = 1295000576
+tensor[13]: name = blk.0.ffn_up.weight, size = 12582912, offset = 1295004672
+tensor[14]: name = blk.1.attn_k.weight, size = 4194304, offset = 1307587584
+tensor[15]: name = blk.1.attn_k_norm.weight, size = 512, offset = 1311781888
+tensor[16]: name = blk.1.attn_norm.weight, size = 4096, offset = 1311782400
+tensor[17]: name = blk.1.attn_output.weight, size = 8388608, offset = 1311786496
+tensor[18]: name = blk.1.attn_q.weight, size = 8388608, offset = 1320175104
+tensor[19]: name = blk.1.attn_q_norm.weight, size = 512, offset = 1328563712
+tensor[20]: name = blk.1.attn_v.weight, size = 4194304, offset = 1328564224
+tensor[21]: name = blk.1.ffn_down.weight, size = 12582912, offset = 1332758528
+tensor[22]: name = blk.1.ffn_gate.weight, size = 12582912, offset = 1345341440
+tensor[23]: name = blk.1.ffn_norm.weight, size = 4096, offset = 1357924352
+tensor[24]: name = blk.1.ffn_up.weight, size = 12582912, offset = 1357928448
+tensor[25]: name = blk.2.attn_k.weight, size = 4194304, offset = 1370511360
+tensor[26]: name = blk.2.attn_k_norm.weight, size = 512, offset = 1374705664
+tensor[27]: name = blk.2.attn_norm.weight, size = 4096, offset = 1374706176
+tensor[28]: name = blk.2.attn_output.weight, size = 8388608, offset = 1374710272
+tensor[29]: name = blk.2.attn_q.weight, size = 8388608, offset = 1383098880
+tensor[30]: name = blk.2.attn_q_norm.weight, size = 512, offset = 1391487488
+tensor[31]: name = blk.2.attn_v.weight, size = 4194304, offset = 1391488000
+tensor[32]: name = blk.2.ffn_down.weight, size = 12582912, offset = 1395682304
+tensor[33]: name = blk.2.ffn_gate.weight, size = 12582912, offset = 1408265216
+tensor[34]: name = blk.2.ffn_norm.weight, size = 4096, offset = 1420848128
+tensor[35]: name = blk.2.ffn_up.weight, size = 12582912, offset = 1420852224
+tensor[36]: name = blk.3.attn_k.weight, size = 4194304, offset = 1433435136
+tensor[37]: name = blk.3.attn_k_norm.weight, size = 512, offset = 1437629440
+tensor[38]: name = blk.3.attn_norm.weight, size = 4096, offset = 1437629952
+tensor[39]: name = blk.3.attn_output.weight, size = 8388608, offset = 1437634048
+tensor[40]: name = blk.3.attn_q.weight, size = 8388608, offset = 1446022656
+tensor[41]: name = blk.3.attn_q_norm.weight, size = 512, offset = 1454411264
+tensor[42]: name = blk.3.attn_v.weight, size = 4194304, offset = 1454411776
+tensor[43]: name = blk.3.ffn_down.weight, size = 12582912, offset = 1458606080
+tensor[44]: name = blk.3.ffn_gate.weight, size = 12582912, offset = 1471188992
+tensor[45]: name = blk.3.ffn_norm.weight, size = 4096, offset = 1483771904
+tensor[46]: name = blk.3.ffn_up.weight, size = 12582912, offset = 1483776000
+tensor[47]: name = blk.4.attn_k.weight, size = 4194304, offset = 1496358912
+tensor[48]: name = blk.4.attn_k_norm.weight, size = 512, offset = 1500553216
+tensor[49]: name = blk.4.attn_norm.weight, size = 4096, offset = 1500553728
+tensor[50]: name = blk.4.attn_output.weight, size = 8388608, offset = 1500557824
+tensor[51]: name = blk.4.attn_q.weight, size = 8388608, offset = 1508946432
+tensor[52]: name = blk.4.attn_q_norm.weight, size = 512, offset = 1517335040
+tensor[53]: name = blk.4.attn_v.weight, size = 4194304, offset = 1517335552
+tensor[54]: name = blk.4.ffn_down.weight, size = 12582912, offset = 1521529856
+tensor[55]: name = blk.4.ffn_gate.weight, size = 12582912, offset = 1534112768
+tensor[56]: name = blk.4.ffn_norm.weight, size = 4096, offset = 1546695680
+tensor[57]: name = blk.4.ffn_up.weight, size = 12582912, offset = 1546699776
+tensor[58]: name = blk.5.attn_k.weight, size = 4194304, offset = 1559282688
+tensor[59]: name = blk.5.attn_k_norm.weight, size = 512, offset = 1563476992
+tensor[60]: name = blk.5.attn_norm.weight, size = 4096, offset = 1563477504
+tensor[61]: name = blk.5.attn_output.weight, size = 8388608, offset = 1563481600
+tensor[62]: name = blk.5.attn_q.weight, size = 8388608, offset = 1571870208
+tensor[63]: name = blk.5.attn_q_norm.weight, size = 512, offset = 1580258816
+tensor[64]: name = blk.5.attn_v.weight, size = 4194304, offset = 1580259328
+tensor[65]: name = blk.5.ffn_down.weight, size = 12582912, offset = 1584453632
+tensor[66]: name = blk.5.ffn_gate.weight, size = 12582912, offset = 1597036544
+tensor[67]: name = blk.5.ffn_norm.weight, size = 4096, offset = 1609619456
+tensor[68]: name = blk.5.ffn_up.weight, size = 12582912, offset = 1609623552
+tensor[69]: name = blk.6.attn_k.weight, size = 4194304, offset = 1622206464
+tensor[70]: name = blk.6.attn_k_norm.weight, size = 512, offset = 1626400768
+tensor[71]: name = blk.6.attn_norm.weight, size = 4096, offset = 1626401280
+tensor[72]: name = blk.6.attn_output.weight, size = 8388608, offset = 1626405376
+tensor[73]: name = blk.6.attn_q.weight, size = 8388608, offset = 1634793984
+tensor[74]: name = blk.6.attn_q_norm.weight, size = 512, offset = 1643182592
+tensor[75]: name = blk.6.attn_v.weight, size = 4194304, offset = 1643183104
+tensor[76]: name = blk.6.ffn_down.weight, size = 12582912, offset = 1647377408
+tensor[77]: name = blk.6.ffn_gate.weight, size = 12582912, offset = 1659960320
+tensor[78]: name = blk.6.ffn_norm.weight, size = 4096, offset = 1672543232
+tensor[79]: name = blk.6.ffn_up.weight, size = 12582912, offset = 1672547328
+tensor[80]: name = blk.7.attn_k.weight, size = 4194304, offset = 1685130240
+tensor[81]: name = blk.7.attn_k_norm.weight, size = 512, offset = 1689324544
+tensor[82]: name = blk.7.attn_norm.weight, size = 4096, offset = 1689325056
+tensor[83]: name = blk.7.attn_output.weight, size = 8388608, offset = 1689329152
+tensor[84]: name = blk.7.attn_q.weight, size = 8388608, offset = 1697717760
+tensor[85]: name = blk.7.attn_q_norm.weight, size = 512, offset = 1706106368
+tensor[86]: name = blk.7.attn_v.weight, size = 4194304, offset = 1706106880
+tensor[87]: name = blk.7.ffn_down.weight, size = 12582912, offset = 1710301184
+tensor[88]: name = blk.7.ffn_gate.weight, size = 12582912, offset = 1722884096
+tensor[89]: name = blk.7.ffn_norm.weight, size = 4096, offset = 1735467008
+tensor[90]: name = blk.7.ffn_up.weight, size = 12582912, offset = 1735471104
+tensor[91]: name = blk.8.attn_k.weight, size = 4194304, offset = 1748054016
+tensor[92]: name = blk.8.attn_k_norm.weight, size = 512, offset = 1752248320
+tensor[93]: name = blk.8.attn_norm.weight, size = 4096, offset = 1752248832
+tensor[94]: name = blk.8.attn_output.weight, size = 8388608, offset = 1752252928
+tensor[95]: name = blk.8.attn_q.weight, size = 8388608, offset = 1760641536
+tensor[96]: name = blk.8.attn_q_norm.weight, size = 512, offset = 1769030144
+tensor[97]: name = blk.8.attn_v.weight, size = 4194304, offset = 1769030656
+tensor[98]: name = blk.8.ffn_down.weight, size = 12582912, offset = 1773224960
+tensor[99]: name = blk.8.ffn_gate.weight, size = 12582912, offset = 1785807872
+tensor[100]: name = blk.8.ffn_norm.weight, size = 4096, offset = 1798390784
+tensor[101]: name = blk.8.ffn_up.weight, size = 12582912, offset = 1798394880
+tensor[102]: name = blk.9.attn_k.weight, size = 4194304, offset = 1810977792
+tensor[103]: name = blk.9.attn_k_norm.weight, size = 512, offset = 1815172096
+tensor[104]: name = blk.9.attn_norm.weight, size = 4096, offset = 1815172608
+tensor[105]: name = blk.9.attn_output.weight, size = 8388608, offset = 1815176704
+tensor[106]: name = blk.9.attn_q.weight, size = 8388608, offset = 1823565312
+tensor[107]: name = blk.9.attn_q_norm.weight, size = 512, offset = 1831953920
+tensor[108]: name = blk.9.attn_v.weight, size = 4194304, offset = 1831954432
+tensor[109]: name = blk.9.ffn_down.weight, size = 12582912, offset = 1836148736
+tensor[110]: name = blk.9.ffn_gate.weight, size = 12582912, offset = 1848731648
+tensor[111]: name = blk.9.ffn_norm.weight, size = 4096, offset = 1861314560
+tensor[112]: name = blk.9.ffn_up.weight, size = 12582912, offset = 1861318656
+tensor[113]: name = blk.10.attn_k.weight, size = 4194304, offset = 1873901568
+tensor[114]: name = blk.10.attn_k_norm.weight, size = 512, offset = 1878095872
+tensor[115]: name = blk.10.attn_norm.weight, size = 4096, offset = 1878096384
+tensor[116]: name = blk.10.attn_output.weight, size = 8388608, offset = 1878100480
+tensor[117]: name = blk.10.attn_q.weight, size = 8388608, offset = 1886489088
+tensor[118]: name = blk.10.attn_q_norm.weight, size = 512, offset = 1894877696
+tensor[119]: name = blk.10.attn_v.weight, size = 4194304, offset = 1894878208
+tensor[120]: name = blk.10.ffn_down.weight, size = 12582912, offset = 1899072512
+tensor[121]: name = blk.10.ffn_gate.weight, size = 12582912, offset = 1911655424
+tensor[122]: name = blk.10.ffn_norm.weight, size = 4096, offset = 1924238336
+tensor[123]: name = blk.10.ffn_up.weight, size = 12582912, offset = 1924242432
+tensor[124]: name = blk.11.attn_k.weight, size = 4194304, offset = 1936825344
+tensor[125]: name = blk.11.attn_k_norm.weight, size = 512, offset = 1941019648
+tensor[126]: name = blk.11.attn_norm.weight, size = 4096, offset = 1941020160
+tensor[127]: name = blk.11.attn_output.weight, size = 8388608, offset = 1941024256
+tensor[128]: name = blk.11.attn_q.weight, size = 8388608, offset = 1949412864
+tensor[129]: name = blk.11.attn_q_norm.weight, size = 512, offset = 1957801472
+tensor[130]: name = blk.11.attn_v.weight, size = 4194304, offset = 1957801984
+tensor[131]: name = blk.11.ffn_down.weight, size = 12582912, offset = 1961996288
+tensor[132]: name = blk.11.ffn_gate.weight, size = 12582912, offset = 1974579200
+tensor[133]: name = blk.11.ffn_norm.weight, size = 4096, offset = 1987162112
+tensor[134]: name = blk.11.ffn_up.weight, size = 12582912, offset = 1987166208
+tensor[135]: name = blk.12.attn_k.weight, size = 4194304, offset = 1999749120
+tensor[136]: name = blk.12.attn_k_norm.weight, size = 512, offset = 2003943424
+tensor[137]: name = blk.12.attn_norm.weight, size = 4096, offset = 2003943936
+tensor[138]: name = blk.12.attn_output.weight, size = 8388608, offset = 2003948032
+tensor[139]: name = blk.12.attn_q.weight, size = 8388608, offset = 2012336640
+tensor[140]: name = blk.12.attn_q_norm.weight, size = 512, offset = 2020725248
+tensor[141]: name = blk.12.attn_v.weight, size = 4194304, offset = 2020725760
+tensor[142]: name = blk.12.ffn_down.weight, size = 12582912, offset = 2024920064
+tensor[143]: name = blk.12.ffn_gate.weight, size = 12582912, offset = 2037502976
+tensor[144]: name = blk.12.ffn_norm.weight, size = 4096, offset = 2050085888
+tensor[145]: name = blk.12.ffn_up.weight, size = 12582912, offset = 2050089984
+tensor[146]: name = blk.13.attn_k.weight, size = 4194304, offset = 2062672896
+tensor[147]: name = blk.13.attn_k_norm.weight, size = 512, offset = 2066867200
+tensor[148]: name = blk.13.attn_norm.weight, size = 4096, offset = 2066867712
+tensor[149]: name = blk.13.attn_output.weight, size = 8388608, offset = 2066871808
+tensor[150]: name = blk.13.attn_q.weight, size = 8388608, offset = 2075260416
+tensor[151]: name = blk.13.attn_q_norm.weight, size = 512, offset = 2083649024
+tensor[152]: name = blk.13.attn_v.weight, size = 4194304, offset = 2083649536
+tensor[153]: name = blk.13.ffn_down.weight, size = 12582912, offset = 2087843840
+tensor[154]: name = blk.13.ffn_gate.weight, size = 12582912, offset = 2100426752
+tensor[155]: name = blk.13.ffn_norm.weight, size = 4096, offset = 2113009664
+tensor[156]: name = blk.13.ffn_up.weight, size = 12582912, offset = 2113013760
+tensor[157]: name = blk.14.attn_k.weight, size = 4194304, offset = 2125596672
+tensor[158]: name = blk.14.attn_k_norm.weight, size = 512, offset = 2129790976
+tensor[159]: name = blk.14.attn_norm.weight, size = 4096, offset = 2129791488
+tensor[160]: name = blk.14.attn_output.weight, size = 8388608, offset = 2129795584
+tensor[161]: name = blk.14.attn_q.weight, size = 8388608, offset = 2138184192
+tensor[162]: name = blk.14.attn_q_norm.weight, size = 512, offset = 2146572800
+tensor[163]: name = blk.14.attn_v.weight, size = 4194304, offset = 2146573312
+tensor[164]: name = blk.14.ffn_down.weight, size = 12582912, offset = 2150767616
+tensor[165]: name = blk.14.ffn_gate.weight, size = 12582912, offset = 2163350528
+tensor[166]: name = blk.14.ffn_norm.weight, size = 4096, offset = 2175933440
+tensor[167]: name = blk.14.ffn_up.weight, size = 12582912, offset = 2175937536
+tensor[168]: name = blk.15.attn_k.weight, size = 4194304, offset = 2188520448
+tensor[169]: name = blk.15.attn_k_norm.weight, size = 512, offset = 2192714752
+tensor[170]: name = blk.15.attn_norm.weight, size = 4096, offset = 2192715264
+tensor[171]: name = blk.15.attn_output.weight, size = 8388608, offset = 2192719360
+tensor[172]: name = blk.15.attn_q.weight, size = 8388608, offset = 2201107968
+tensor[173]: name = blk.15.attn_q_norm.weight, size = 512, offset = 2209496576
+tensor[174]: name = blk.15.attn_v.weight, size = 4194304, offset = 2209497088
+tensor[175]: name = blk.15.ffn_down.weight, size = 12582912, offset = 2213691392
+tensor[176]: name = blk.15.ffn_gate.weight, size = 12582912, offset = 2226274304
+tensor[177]: name = blk.15.ffn_norm.weight, size = 4096, offset = 2238857216
+tensor[178]: name = blk.15.ffn_up.weight, size = 12582912, offset = 2238861312
+tensor[179]: name = blk.16.attn_k.weight, size = 4194304, offset = 2251444224
+tensor[180]: name = blk.16.attn_k_norm.weight, size = 512, offset = 2255638528
+tensor[181]: name = blk.16.attn_norm.weight, size = 4096, offset = 2255639040
+tensor[182]: name = blk.16.attn_output.weight, size = 8388608, offset = 2255643136
+tensor[183]: name = blk.16.attn_q.weight, size = 8388608, offset = 2264031744
+tensor[184]: name = blk.16.attn_q_norm.weight, size = 512, offset = 2272420352
+tensor[185]: name = blk.16.attn_v.weight, size = 4194304, offset = 2272420864
+tensor[186]: name = blk.16.ffn_down.weight, size = 12582912, offset = 2276615168
+tensor[187]: name = blk.16.ffn_gate.weight, size = 12582912, offset = 2289198080
+tensor[188]: name = blk.16.ffn_norm.weight, size = 4096, offset = 2301780992
+tensor[189]: name = blk.16.ffn_up.weight, size = 12582912, offset = 2301785088
+tensor[190]: name = blk.17.attn_k.weight, size = 4194304, offset = 2314368000
+tensor[191]: name = blk.17.attn_k_norm.weight, size = 512, offset = 2318562304
+tensor[192]: name = blk.17.attn_norm.weight, size = 4096, offset = 2318562816
+tensor[193]: name = blk.17.attn_output.weight, size = 8388608, offset = 2318566912
+tensor[194]: name = blk.17.attn_q.weight, size = 8388608, offset = 2326955520
+tensor[195]: name = blk.17.attn_q_norm.weight, size = 512, offset = 2335344128
+tensor[196]: name = blk.17.attn_v.weight, size = 4194304, offset = 2335344640
+tensor[197]: name = blk.17.ffn_down.weight, size = 12582912, offset = 2339538944
+tensor[198]: name = blk.17.ffn_gate.weight, size = 12582912, offset = 2352121856
+tensor[199]: name = blk.17.ffn_norm.weight, size = 4096, offset = 2364704768
+tensor[200]: name = blk.17.ffn_up.weight, size = 12582912, offset = 2364708864
+tensor[201]: name = blk.18.attn_k.weight, size = 4194304, offset = 2377291776
+tensor[202]: name = blk.18.attn_k_norm.weight, size = 512, offset = 2381486080
+tensor[203]: name = blk.18.attn_norm.weight, size = 4096, offset = 2381486592
+tensor[204]: name = blk.18.attn_output.weight, size = 8388608, offset = 2381490688
+tensor[205]: name = blk.18.attn_q.weight, size = 8388608, offset = 2389879296
+tensor[206]: name = blk.18.attn_q_norm.weight, size = 512, offset = 2398267904
+tensor[207]: name = blk.18.attn_v.weight, size = 4194304, offset = 2398268416
+tensor[208]: name = blk.18.ffn_down.weight, size = 12582912, offset = 2402462720
+tensor[209]: name = blk.18.ffn_gate.weight, size = 12582912, offset = 2415045632
+tensor[210]: name = blk.18.ffn_norm.weight, size = 4096, offset = 2427628544
+tensor[211]: name = blk.18.ffn_up.weight, size = 12582912, offset = 2427632640
+tensor[212]: name = blk.19.attn_k.weight, size = 4194304, offset = 2440215552
+tensor[213]: name = blk.19.attn_k_norm.weight, size = 512, offset = 2444409856
+tensor[214]: name = blk.19.attn_norm.weight, size = 4096, offset = 2444410368
+tensor[215]: name = blk.19.attn_output.weight, size = 8388608, offset = 2444414464
+tensor[216]: name = blk.19.attn_q.weight, size = 8388608, offset = 2452803072
+tensor[217]: name = blk.19.attn_q_norm.weight, size = 512, offset = 2461191680
+tensor[218]: name = blk.19.attn_v.weight, size = 4194304, offset = 2461192192
+tensor[219]: name = blk.19.ffn_down.weight, size = 12582912, offset = 2465386496
+tensor[220]: name = blk.19.ffn_gate.weight, size = 12582912, offset = 2477969408
+tensor[221]: name = blk.19.ffn_norm.weight, size = 4096, offset = 2490552320
+tensor[222]: name = blk.19.ffn_up.weight, size = 12582912, offset = 2490556416
+tensor[223]: name = blk.20.attn_k.weight, size = 4194304, offset = 2503139328
+tensor[224]: name = blk.20.attn_k_norm.weight, size = 512, offset = 2507333632
+tensor[225]: name = blk.20.attn_norm.weight, size = 4096, offset = 2507334144
+tensor[226]: name = blk.20.attn_output.weight, size = 8388608, offset = 2507338240
+tensor[227]: name = blk.20.attn_q.weight, size = 8388608, offset = 2515726848
+tensor[228]: name = blk.20.attn_q_norm.weight, size = 512, offset = 2524115456
+tensor[229]: name = blk.20.attn_v.weight, size = 4194304, offset = 2524115968
+tensor[230]: name = blk.20.ffn_down.weight, size = 12582912, offset = 2528310272
+tensor[231]: name = blk.20.ffn_gate.weight, size = 12582912, offset = 2540893184
+tensor[232]: name = blk.20.ffn_norm.weight, size = 4096, offset = 2553476096
+tensor[233]: name = blk.20.ffn_up.weight, size = 12582912, offset = 2553480192
+tensor[234]: name = blk.21.attn_k.weight, size = 4194304, offset = 2566063104
+tensor[235]: name = blk.21.attn_k_norm.weight, size = 512, offset = 2570257408
+tensor[236]: name = blk.21.attn_norm.weight, size = 4096, offset = 2570257920
+tensor[237]: name = blk.21.attn_output.weight, size = 8388608, offset = 2570262016
+tensor[238]: name = blk.21.attn_q.weight, size = 8388608, offset = 2578650624
+tensor[239]: name = blk.21.attn_q_norm.weight, size = 512, offset = 2587039232
+tensor[240]: name = blk.21.attn_v.weight, size = 4194304, offset = 2587039744
+tensor[241]: name = blk.21.ffn_down.weight, size = 12582912, offset = 2591234048
+tensor[242]: name = blk.21.ffn_gate.weight, size = 12582912, offset = 2603816960
+tensor[243]: name = blk.21.ffn_norm.weight, size = 4096, offset = 2616399872
+tensor[244]: name = blk.21.ffn_up.weight, size = 12582912, offset = 2616403968
+tensor[245]: name = blk.22.attn_k.weight, size = 4194304, offset = 2628986880
+tensor[246]: name = blk.22.attn_k_norm.weight, size = 512, offset = 2633181184
+tensor[247]: name = blk.22.attn_norm.weight, size = 4096, offset = 2633181696
+tensor[248]: name = blk.22.attn_output.weight, size = 8388608, offset = 2633185792
+tensor[249]: name = blk.22.attn_q.weight, size = 8388608, offset = 2641574400
+tensor[250]: name = blk.22.attn_q_norm.weight, size = 512, offset = 2649963008
+tensor[251]: name = blk.22.attn_v.weight, size = 4194304, offset = 2649963520
+tensor[252]: name = blk.22.ffn_down.weight, size = 12582912, offset = 2654157824
+tensor[253]: name = blk.22.ffn_gate.weight, size = 12582912, offset = 2666740736
+tensor[254]: name = blk.22.ffn_norm.weight, size = 4096, offset = 2679323648
+tensor[255]: name = blk.22.ffn_up.weight, size = 12582912, offset = 2679327744
+tensor[256]: name = blk.23.attn_k.weight, size = 4194304, offset = 2691910656
+tensor[257]: name = blk.23.attn_k_norm.weight, size = 512, offset = 2696104960
+tensor[258]: name = blk.23.attn_norm.weight, size = 4096, offset = 2696105472
+tensor[259]: name = blk.23.attn_output.weight, size = 8388608, offset = 2696109568
+tensor[260]: name = blk.23.attn_q.weight, size = 8388608, offset = 2704498176
+tensor[261]: name = blk.23.attn_q_norm.weight, size = 512, offset = 2712886784
+tensor[262]: name = blk.23.attn_v.weight, size = 4194304, offset = 2712887296
+tensor[263]: name = blk.23.ffn_down.weight, size = 12582912, offset = 2717081600
+tensor[264]: name = blk.23.ffn_gate.weight, size = 12582912, offset = 2729664512
+tensor[265]: name = blk.23.ffn_norm.weight, size = 4096, offset = 2742247424
+tensor[266]: name = blk.23.ffn_up.weight, size = 12582912, offset = 2742251520
+tensor[267]: name = blk.24.attn_k.weight, size = 4194304, offset = 2754834432
+tensor[268]: name = blk.24.attn_k_norm.weight, size = 512, offset = 2759028736
+tensor[269]: name = blk.24.attn_norm.weight, size = 4096, offset = 2759029248
+tensor[270]: name = blk.24.attn_output.weight, size = 8388608, offset = 2759033344
+tensor[271]: name = blk.24.attn_q.weight, size = 8388608, offset = 2767421952
+tensor[272]: name = blk.24.attn_q_norm.weight, size = 512, offset = 2775810560
+tensor[273]: name = blk.24.attn_v.weight, size = 4194304, offset = 2775811072
+tensor[274]: name = blk.24.ffn_down.weight, size = 12582912, offset = 2780005376
+tensor[275]: name = blk.24.ffn_gate.weight, size = 12582912, offset = 2792588288
+tensor[276]: name = blk.24.ffn_norm.weight, size = 4096, offset = 2805171200
+tensor[277]: name = blk.24.ffn_up.weight, size = 12582912, offset = 2805175296
+tensor[278]: name = blk.25.attn_k.weight, size = 4194304, offset = 2817758208
+tensor[279]: name = blk.25.attn_k_norm.weight, size = 512, offset = 2821952512
+tensor[280]: name = blk.25.attn_norm.weight, size = 4096, offset = 2821953024
+tensor[281]: name = blk.25.attn_output.weight, size = 8388608, offset = 2821957120
+tensor[282]: name = blk.25.attn_q.weight, size = 8388608, offset = 2830345728
+tensor[283]: name = blk.25.attn_q_norm.weight, size = 512, offset = 2838734336
+tensor[284]: name = blk.25.attn_v.weight, size = 4194304, offset = 2838734848
+tensor[285]: name = blk.25.ffn_down.weight, size = 12582912, offset = 2842929152
+tensor[286]: name = blk.25.ffn_gate.weight, size = 12582912, offset = 2855512064
+tensor[287]: name = blk.25.ffn_norm.weight, size = 4096, offset = 2868094976
+tensor[288]: name = blk.25.ffn_up.weight, size = 12582912, offset = 2868099072
+tensor[289]: name = blk.26.attn_k.weight, size = 4194304, offset = 2880681984
+tensor[290]: name = blk.26.attn_k_norm.weight, size = 512, offset = 2884876288
+tensor[291]: name = blk.26.attn_norm.weight, size = 4096, offset = 2884876800
+tensor[292]: name = blk.26.attn_output.weight, size = 8388608, offset = 2884880896
+tensor[293]: name = blk.26.attn_q.weight, size = 8388608, offset = 2893269504
+tensor[294]: name = blk.26.attn_q_norm.weight, size = 512, offset = 2901658112
+tensor[295]: name = blk.26.attn_v.weight, size = 4194304, offset = 2901658624
+tensor[296]: name = blk.26.ffn_down.weight, size = 12582912, offset = 2905852928
+tensor[297]: name = blk.26.ffn_gate.weight, size = 12582912, offset = 2918435840
+tensor[298]: name = blk.26.ffn_norm.weight, size = 4096, offset = 2931018752
+tensor[299]: name = blk.26.ffn_up.weight, size = 12582912, offset = 2931022848
+tensor[300]: name = blk.27.attn_k.weight, size = 4194304, offset = 2943605760
+tensor[301]: name = blk.27.attn_k_norm.weight, size = 512, offset = 2947800064
+tensor[302]: name = blk.27.attn_norm.weight, size = 4096, offset = 2947800576
+tensor[303]: name = blk.27.attn_output.weight, size = 8388608, offset = 2947804672
+tensor[304]: name = blk.27.attn_q.weight, size = 8388608, offset = 2956193280
+tensor[305]: name = blk.27.attn_q_norm.weight, size = 512, offset = 2964581888
+tensor[306]: name = blk.27.attn_v.weight, size = 4194304, offset = 2964582400
+tensor[307]: name = blk.27.ffn_down.weight, size = 12582912, offset = 2968776704
+tensor[308]: name = blk.27.ffn_gate.weight, size = 12582912, offset = 2981359616
+tensor[309]: name = blk.27.ffn_norm.weight, size = 4096, offset = 2993942528
+tensor[310]: name = blk.27.ffn_up.weight, size = 12582912, offset = 2993946624