mirror of
https://github.com/linkedin/Liger-Kernel.git
synced 2026-10-12 02:17:47 +00:00
Restore benchmarks after gh-deploy
This commit is contained in:
github-actions[bot]
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,55 @@
|
||||
kernel_name,kernel_provider,kernel_operation_mode,metric_name,metric_unit,x_name,x_label,x_value,y_value_50,y_value_20,y_value_80,extra_benchmark_config_str,gpu_name,timestamp,liger_version
|
||||
kto_loss,liger,forward,speed,ms,B,Batch Size (B),2,3.2825279235839844,3.2804479598999023,3.2853888988494875,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:19:21,0.5.10
|
||||
kto_loss,liger,forward,speed,ms,B,Batch Size (B),4,6.605775833129883,6.478592014312744,6.648127937316895,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:19:21,0.5.10
|
||||
kto_loss,liger,forward,speed,ms,B,Batch Size (B),8,12.956992149353027,12.9052095413208,12.996537399291991,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:19:21,0.5.10
|
||||
kto_loss,liger,forward,speed,ms,B,Batch Size (B),16,25.663455963134766,25.593625259399417,25.742118072509765,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:19:21,0.5.10
|
||||
kto_loss,liger,forward,speed,ms,B,Batch Size (B),32,51.20742416381836,51.20742416381836,51.20742416381836,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:19:21,0.5.10
|
||||
kto_loss,huggingface,forward,speed,ms,B,Batch Size (B),2,3.5336320400238037,3.5222464084625247,3.544422483444214,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:19:38,0.5.10
|
||||
kto_loss,huggingface,forward,speed,ms,B,Batch Size (B),4,6.366528034210205,6.352678298950195,6.390655994415283,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:19:38,0.5.10
|
||||
kto_loss,huggingface,forward,speed,ms,B,Batch Size (B),8,12.048687934875488,12.035417366027833,12.061068725585937,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:19:38,0.5.10
|
||||
kto_loss,huggingface,forward,speed,ms,B,Batch Size (B),16,23.290640830993652,23.276774978637697,23.311956024169923,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:19:38,0.5.10
|
||||
kto_loss,huggingface,forward,speed,ms,B,Batch Size (B),32,46.10256004333496,46.08627853393555,46.118841552734374,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:19:38,0.5.10
|
||||
kto_loss,liger,full,speed,ms,B,Batch Size (B),2,5.123855829238892,4.941535949707031,5.643481731414796,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:19:53,0.5.10
|
||||
kto_loss,liger,full,speed,ms,B,Batch Size (B),4,8.228639602661133,8.044639587402344,8.611104011535645,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:19:53,0.5.10
|
||||
kto_loss,liger,full,speed,ms,B,Batch Size (B),8,14.871295928955078,14.519136428833008,15.598176002502441,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:19:53,0.5.10
|
||||
kto_loss,liger,full,speed,ms,B,Batch Size (B),16,27.374431610107422,27.2985538482666,27.46200294494629,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:19:53,0.5.10
|
||||
kto_loss,liger,full,speed,ms,B,Batch Size (B),32,53.22422409057617,53.22422409057617,53.22422409057617,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:19:53,0.5.10
|
||||
kto_loss,huggingface,full,speed,ms,B,Batch Size (B),2,5.69536018371582,5.379321479797364,6.10686092376709,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:20:10,0.5.10
|
||||
kto_loss,huggingface,full,speed,ms,B,Batch Size (B),4,9.326096057891846,9.312140464782715,9.343622016906739,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:20:10,0.5.10
|
||||
kto_loss,huggingface,full,speed,ms,B,Batch Size (B),8,17.51139259338379,17.49594192504883,17.544064331054688,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:20:10,0.5.10
|
||||
kto_loss,huggingface,full,speed,ms,B,Batch Size (B),16,33.48164749145508,33.46792831420898,33.49536666870117,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:20:10,0.5.10
|
||||
kto_loss,huggingface,full,speed,ms,B,Batch Size (B),32,65.69068908691406,65.69068908691406,65.69068908691406,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:20:10,0.5.10
|
||||
kto_loss,liger,full,memory,MB,B,Batch Size (B),2,2077.2275390625,2077.2275390625,2077.2275390625,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:20:36,0.5.10
|
||||
kto_loss,liger,full,memory,MB,B,Batch Size (B),4,2083.2353515625,2083.2353515625,2083.2353515625,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:20:36,0.5.10
|
||||
kto_loss,liger,full,memory,MB,B,Batch Size (B),8,2095.2509765625,2095.2509765625,2095.2509765625,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:20:36,0.5.10
|
||||
kto_loss,liger,full,memory,MB,B,Batch Size (B),16,2119.2822265625,2119.2822265625,2119.2822265625,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:20:36,0.5.10
|
||||
kto_loss,liger,full,memory,MB,B,Batch Size (B),32,2166.5947265625,2166.5947265625,2166.5947265625,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:20:36,0.5.10
|
||||
kto_loss,huggingface,full,memory,MB,B,Batch Size (B),2,3077.24462890625,3077.24462890625,3077.24462890625,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:20:51,0.5.10
|
||||
kto_loss,huggingface,full,memory,MB,B,Batch Size (B),4,4834.26123046875,4834.26123046875,4834.26123046875,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:20:51,0.5.10
|
||||
kto_loss,huggingface,full,memory,MB,B,Batch Size (B),8,8347.794921875,8347.794921875,8347.794921875,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:20:51,0.5.10
|
||||
kto_loss,huggingface,full,memory,MB,B,Batch Size (B),16,15378.361328125,15378.361328125,15378.361328125,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:20:51,0.5.10
|
||||
kto_loss,huggingface,full,memory,MB,B,Batch Size (B),32,29437.994140625,29437.994140625,29437.994140625,"{""T"": 512, ""H"": 1024, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16"", ""bias"": true, ""beta"": 0.1, ""ignore_index"": 42}",NVIDIA H200,2025-06-12 18:20:51,0.5.10
|
||||
fused_linear_cpo_loss,liger,forward,speed,ms,B,B,2,2898.949951171875,2898.949951171875,2898.949951171875,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:23:06,0.5.10
|
||||
fused_linear_cpo_loss,liger,forward,speed,ms,B,B,4,2889.01318359375,2889.01318359375,2889.01318359375,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:23:06,0.5.10
|
||||
fused_linear_cpo_loss,liger,forward,speed,ms,B,B,8,2910.042724609375,2910.042724609375,2910.042724609375,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:23:06,0.5.10
|
||||
fused_linear_cpo_loss,liger,forward,speed,ms,B,B,16,2893.47265625,2893.47265625,2893.47265625,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:23:06,0.5.10
|
||||
fused_linear_cpo_loss,huggingface,forward,speed,ms,B,B,2,2897.0673828125,2897.0673828125,2897.0673828125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:24:39,0.5.10
|
||||
fused_linear_cpo_loss,huggingface,forward,speed,ms,B,B,4,2916.94091796875,2916.94091796875,2916.94091796875,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:24:39,0.5.10
|
||||
fused_linear_cpo_loss,huggingface,forward,speed,ms,B,B,8,2896.0009765625,2896.0009765625,2896.0009765625,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:24:39,0.5.10
|
||||
fused_linear_cpo_loss,huggingface,forward,speed,ms,B,B,16,2913.238525390625,2913.238525390625,2913.238525390625,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:24:39,0.5.10
|
||||
fused_linear_cpo_loss,liger,full,speed,ms,B,B,2,2897.10791015625,2897.10791015625,2897.10791015625,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:26:13,0.5.10
|
||||
fused_linear_cpo_loss,liger,full,speed,ms,B,B,4,2922.138671875,2922.138671875,2922.138671875,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:26:13,0.5.10
|
||||
fused_linear_cpo_loss,liger,full,speed,ms,B,B,8,2951.142822265625,2951.142822265625,2951.142822265625,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:26:13,0.5.10
|
||||
fused_linear_cpo_loss,liger,full,speed,ms,B,B,16,2996.000244140625,2996.000244140625,2996.000244140625,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:26:13,0.5.10
|
||||
fused_linear_cpo_loss,huggingface,full,speed,ms,B,B,2,2903.344970703125,2903.344970703125,2903.344970703125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:27:47,0.5.10
|
||||
fused_linear_cpo_loss,huggingface,full,speed,ms,B,B,4,2921.09912109375,2921.09912109375,2921.09912109375,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:27:47,0.5.10
|
||||
fused_linear_cpo_loss,huggingface,full,speed,ms,B,B,8,2906.561767578125,2906.561767578125,2906.561767578125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:27:47,0.5.10
|
||||
fused_linear_cpo_loss,huggingface,full,speed,ms,B,B,16,2952.124267578125,2952.124267578125,2952.124267578125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:27:47,0.5.10
|
||||
fused_linear_cpo_loss,liger,full,memory,MB,B,B,2,4138.03466796875,4138.03466796875,4138.03466796875,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:29:45,0.5.10
|
||||
fused_linear_cpo_loss,liger,full,memory,MB,B,B,4,4186.0517578125,4186.0517578125,4186.0517578125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:29:45,0.5.10
|
||||
fused_linear_cpo_loss,liger,full,memory,MB,B,B,8,4282.0859375,4282.0859375,4282.0859375,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:29:45,0.5.10
|
||||
fused_linear_cpo_loss,liger,full,memory,MB,B,B,16,4474.154296875,4474.154296875,4474.154296875,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:29:45,0.5.10
|
||||
fused_linear_cpo_loss,huggingface,full,memory,MB,B,B,2,4606.017578125,4606.017578125,4606.017578125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:31:41,0.5.10
|
||||
fused_linear_cpo_loss,huggingface,full,memory,MB,B,B,4,8144.033203125,8144.033203125,8144.033203125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:31:41,0.5.10
|
||||
fused_linear_cpo_loss,huggingface,full,memory,MB,B,B,8,15222.064453125,15222.064453125,15222.064453125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:31:41,0.5.10
|
||||
fused_linear_cpo_loss,huggingface,full,memory,MB,B,B,16,29378.126953125,29378.126953125,29378.126953125,"{""T"": 1024, ""H"": 4096, ""V"": 128256, ""mode"": ""forward"", ""dtype"": ""torch.bfloat16""}",NVIDIA H200,2025-06-12 18:31:41,0.5.10
|
||||
|
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,34 @@
|
||||
b6ed735
|
||||
c1bdc24
|
||||
0701bab
|
||||
d47751c
|
||||
08906d3
|
||||
02bac05
|
||||
5e8f12b
|
||||
7705dcc
|
||||
2845fe8
|
||||
1c07943
|
||||
77a4c1a
|
||||
c856fba
|
||||
33924d2
|
||||
d5648bf-v0.6.3
|
||||
a80f30b
|
||||
817436f
|
||||
4c32ab6
|
||||
0a62700-v0.6.4
|
||||
0a62700
|
||||
3d075fd
|
||||
49e6353
|
||||
6059dfb
|
||||
5101e3c
|
||||
3b79375
|
||||
4444fdf
|
||||
7656bcb
|
||||
81f932a-v0.6.5
|
||||
1fdb668
|
||||
7644a0f-v0.7.0
|
||||
7644a0f
|
||||
9c1ddb7
|
||||
5057ca8
|
||||
e6f84b8
|
||||
ab7fe2a
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,55 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0"/>
|
||||
<title>Benchmark Visualization</title>
|
||||
<link rel="stylesheet" href="style.css"/>
|
||||
<script src="https://cdn.jsdelivr.net/npm/chart.js"></script>
|
||||
<script src="https://cdn.jsdelivr.net/npm/[email protected]/papaparse.min.js"></script>
|
||||
</head>
|
||||
<body>
|
||||
<div class="container">
|
||||
<div class="panel1">
|
||||
<h1>Panel 1</h1>
|
||||
<div class="controls">
|
||||
<label>Commit Hash:</label>
|
||||
<select id="commit"></select>
|
||||
|
||||
<label for="kernel">Kernel Name:</label>
|
||||
<select id="kernel"></select>
|
||||
|
||||
<label for="mode">Pass Type:</label>
|
||||
<select id="mode"></select>
|
||||
</div>
|
||||
<div class="metric-section">
|
||||
<h2>Speed Metrics</h2>
|
||||
<canvas id="benchmarkChartSpeed"></canvas>
|
||||
<h2>Memory Metrics</h2>
|
||||
<canvas id="benchmarkChartMemory"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
<div class="panel2">
|
||||
<h1>Panel 2</h1>
|
||||
<div class="controls">
|
||||
<label>Commit Hash:</label>
|
||||
<select id="commit2"></select>
|
||||
|
||||
<label for="kernel">Kernel Name:</label>
|
||||
<select id="kernel2"></select>
|
||||
|
||||
<label for="mode">Pass Type:</label>
|
||||
<select id="mode2"></select>
|
||||
</div>
|
||||
<div class="metric-section">
|
||||
<h2>Speed Metrics</h2>
|
||||
<canvas id="benchmarkChart2Speed"></canvas>
|
||||
<h2>Memory Metrics</h2>
|
||||
<canvas id="benchmarkChart2Memory"></canvas>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="script.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,295 @@
|
||||
let chartSpeed;
|
||||
let chartMemory;
|
||||
let chart2Speed;
|
||||
let chart2Memory;
|
||||
|
||||
const primaryBase = 'https://raw.githubusercontent.com/linkedin/Liger-Kernel/refs/heads/gh-pages/benchmarks';
|
||||
const fallbackBase = 'https://cdn.jsdelivr.net/gh/linkedin/Liger-Kernel@gh-pages/benchmarks';
|
||||
|
||||
async function fetchWithFallback(path) {
|
||||
const urls = [
|
||||
`${primaryBase}/${path}`,
|
||||
`${fallbackBase}/${path}`,
|
||||
];
|
||||
let lastErr;
|
||||
for (const url of urls) {
|
||||
try {
|
||||
const res = await fetch(url);
|
||||
if (res.ok) return res;
|
||||
lastErr = new Error(`HTTP ${res.status} for ${url}`);
|
||||
} catch (e) {
|
||||
lastErr = e;
|
||||
}
|
||||
}
|
||||
throw lastErr || new Error('Failed to fetch with fallback');
|
||||
}
|
||||
|
||||
document.addEventListener("DOMContentLoaded", async () => {
|
||||
const commits = await fetchCommitHashes();
|
||||
// const commits = ["a96bdc1", "bc105fd", "cbe66ed"]
|
||||
populateCommitDropdown(commits, "commit");
|
||||
populateCommitDropdown(commits, "commit2");
|
||||
|
||||
document.getElementById("commit").addEventListener("change", (e) => {
|
||||
loadCSV(e.target.value, 1);
|
||||
});
|
||||
|
||||
document.getElementById("commit2").addEventListener("change", (e) => {
|
||||
loadCSV(e.target.value, 2);
|
||||
});
|
||||
|
||||
if (commits.length > 0) {
|
||||
loadCSV(commits[0], 1);
|
||||
loadCSV(commits[0], 2);
|
||||
}
|
||||
});
|
||||
|
||||
async function fetchCommitHashes() {
|
||||
try {
|
||||
const response = await fetchWithFallback('commits.txt');
|
||||
const text = await response.text();
|
||||
// Split by newlines and filter out empty lines
|
||||
return text.split('\n')
|
||||
.map(line => line.trim())
|
||||
.filter(line => line.length > 0);
|
||||
} catch (err) {
|
||||
console.error("Failed to fetch commit hashes:", err);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
function populateCommitDropdown(commits, selectId) {
|
||||
const select = document.getElementById(selectId);
|
||||
select.innerHTML = "";
|
||||
commits.forEach(commit => {
|
||||
const opt = document.createElement("option");
|
||||
opt.value = opt.textContent = commit;
|
||||
select.appendChild(opt);
|
||||
});
|
||||
}
|
||||
|
||||
function loadCSV(commit, panel) {
|
||||
Papa.parse(`${primaryBase}/${commit}/benchmark.csv`, {
|
||||
download: true,
|
||||
header: true,
|
||||
dynamicTyping: true,
|
||||
complete: async (result) => {
|
||||
// If primary failed to load any data rows, try fallback
|
||||
let data = result.data.filter(d => d.kernel_provider);
|
||||
if (!data || data.length === 0) {
|
||||
try {
|
||||
const res = await fetchWithFallback(`${commit}/benchmark.csv`);
|
||||
const text = await res.text();
|
||||
const parsed = Papa.parse(text, { header: true, dynamicTyping: true });
|
||||
data = parsed.data.filter(d => d.kernel_provider);
|
||||
} catch (e) {
|
||||
alert("Failed to load CSV for commit: " + commit);
|
||||
console.error(e);
|
||||
return;
|
||||
}
|
||||
}
|
||||
if (panel === 1) {
|
||||
setupControls(data, 1);
|
||||
renderCharts(data, 1);
|
||||
} else {
|
||||
setupControls(data, 2);
|
||||
renderCharts(data, 2);
|
||||
}
|
||||
},
|
||||
error: async (err) => {
|
||||
// Try fallback immediately
|
||||
try {
|
||||
const res = await fetchWithFallback(`${commit}/benchmark.csv`);
|
||||
const text = await res.text();
|
||||
const parsed = Papa.parse(text, { header: true, dynamicTyping: true });
|
||||
const data = parsed.data.filter(d => d.kernel_provider);
|
||||
if (panel === 1) {
|
||||
setupControls(data, 1);
|
||||
renderCharts(data, 1);
|
||||
} else {
|
||||
setupControls(data, 2);
|
||||
renderCharts(data, 2);
|
||||
}
|
||||
} catch (e) {
|
||||
alert("Failed to load CSV for commit: " + commit);
|
||||
console.error(err);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function setupControls(data, panel) {
|
||||
const kernelSet = new Set(data.map(d => d.kernel_name));
|
||||
const modeSet = new Set(data.map(d => d.kernel_operation_mode));
|
||||
|
||||
const kernelSelect = document.getElementById(panel === 1 ? "kernel" : "kernel2");
|
||||
const modeSelect = document.getElementById(panel === 1 ? "mode" : "mode2");
|
||||
|
||||
kernelSelect.innerHTML = "";
|
||||
modeSelect.innerHTML = "";
|
||||
|
||||
kernelSet.forEach(k => {
|
||||
const opt = document.createElement("option");
|
||||
opt.value = opt.textContent = k;
|
||||
kernelSelect.appendChild(opt);
|
||||
});
|
||||
|
||||
modeSet.forEach(m => {
|
||||
const opt = document.createElement("option");
|
||||
opt.value = opt.textContent = m;
|
||||
modeSelect.appendChild(opt);
|
||||
});
|
||||
|
||||
kernelSelect.addEventListener("change", () => renderCharts(data, panel));
|
||||
modeSelect.addEventListener("change", () => renderCharts(data, panel));
|
||||
}
|
||||
|
||||
function renderCharts(data, panel) {
|
||||
const kernel = document.getElementById(panel === 1 ? "kernel" : "kernel2").value;
|
||||
const mode = document.getElementById(panel === 1 ? "mode" : "mode2").value;
|
||||
|
||||
// Render speed chart
|
||||
renderChart(data, panel, "speed", kernel, mode);
|
||||
// Render memory chart
|
||||
renderChart(data, panel, "memory", kernel, mode);
|
||||
}
|
||||
|
||||
function renderChart(data, panel, metric, kernel, mode) {
|
||||
|
||||
const filtered = data.filter(
|
||||
d =>
|
||||
d.kernel_name === kernel &&
|
||||
d.metric_name.toLowerCase() === metric.toLowerCase() &&
|
||||
d.kernel_operation_mode === mode
|
||||
);
|
||||
|
||||
|
||||
const batchSizes = [...new Set(filtered.map(d => d.x_value))].sort((a, b) => a - b);
|
||||
const providers = [...new Set(filtered.map(d => d.kernel_provider))];
|
||||
|
||||
|
||||
|
||||
const datasets = providers.map(provider => {
|
||||
const values = batchSizes.map(bs => {
|
||||
const row = filtered.find(d => d.kernel_provider === provider && d.x_value === bs);
|
||||
return row ? row.y_value_50 : null;
|
||||
});
|
||||
|
||||
let borderColor;
|
||||
if (provider === "liger") {
|
||||
borderColor = "#FF6B35";
|
||||
} else if (provider === "huggingface") {
|
||||
borderColor = "#4A90E2";
|
||||
} else {
|
||||
borderColor = "#2ECC71"; // For any third provider (like torch_compile)
|
||||
}
|
||||
|
||||
return {
|
||||
label: provider,
|
||||
data: values,
|
||||
borderColor: borderColor,
|
||||
backgroundColor: "transparent",
|
||||
tension: 0.2,
|
||||
pointRadius: 4,
|
||||
pointHoverRadius: 6,
|
||||
borderWidth: 2,
|
||||
};
|
||||
});
|
||||
|
||||
const chartId = panel === 1
|
||||
? (metric === "speed" ? "benchmarkChartSpeed" : "benchmarkChartMemory")
|
||||
: (metric === "speed" ? "benchmarkChart2Speed" : "benchmarkChart2Memory");
|
||||
|
||||
const ctx = document.getElementById(chartId).getContext("2d");
|
||||
|
||||
// Destroy existing chart if it exists
|
||||
if (panel === 1) {
|
||||
if (metric === "speed" && chartSpeed) chartSpeed.destroy();
|
||||
if (metric === "memory" && chartMemory) chartMemory.destroy();
|
||||
} else {
|
||||
if (metric === "speed" && chart2Speed) chart2Speed.destroy();
|
||||
if (metric === "memory" && chart2Memory) chart2Memory.destroy();
|
||||
}
|
||||
|
||||
// Calculate smart Y-axis range for better visibility of differences
|
||||
const allValues = datasets.flatMap(d => d.data.filter(v => v !== null));
|
||||
let yAxisConfig = {
|
||||
title: {
|
||||
display: true,
|
||||
text: `${metric} (${getMetricUnit(metric)})`
|
||||
},
|
||||
type: 'linear'
|
||||
};
|
||||
|
||||
if (allValues.length > 0) {
|
||||
const minVal = Math.min(...allValues);
|
||||
const maxVal = Math.max(...allValues);
|
||||
const range = maxVal - minVal;
|
||||
|
||||
// If the range is small compared to the minimum value, focus on the data range
|
||||
if (range < minVal * 0.1) {
|
||||
yAxisConfig.min = minVal - (range * 0.2);
|
||||
yAxisConfig.max = maxVal + (range * 0.2);
|
||||
} else {
|
||||
yAxisConfig.beginAtZero = true;
|
||||
}
|
||||
} else {
|
||||
yAxisConfig.beginAtZero = true;
|
||||
}
|
||||
|
||||
const newChart = new Chart(ctx, {
|
||||
type: "line",
|
||||
data: {
|
||||
labels: batchSizes,
|
||||
datasets: datasets,
|
||||
},
|
||||
options: {
|
||||
responsive: true,
|
||||
plugins: {
|
||||
title: {
|
||||
display: true,
|
||||
text: `${kernel} - ${metric} (${mode})`,
|
||||
font: {
|
||||
size: 14,
|
||||
weight: 'bold'
|
||||
}
|
||||
},
|
||||
legend: {
|
||||
position: "top",
|
||||
},
|
||||
tooltip: {
|
||||
callbacks: {
|
||||
label: function(context) {
|
||||
const label = context.dataset.label || '';
|
||||
const value = context.parsed.y;
|
||||
return `${label}: ${value.toFixed(2)} ${getMetricUnit(metric)}`;
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
scales: {
|
||||
x: {
|
||||
title: { display: true, text: "Batch Size (B)" },
|
||||
},
|
||||
y: yAxisConfig
|
||||
},
|
||||
interaction: {
|
||||
intersect: false,
|
||||
mode: 'index'
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Store the chart reference
|
||||
if (panel === 1) {
|
||||
if (metric === "speed") chartSpeed = newChart;
|
||||
else chartMemory = newChart;
|
||||
} else {
|
||||
if (metric === "speed") chart2Speed = newChart;
|
||||
else chart2Memory = newChart;
|
||||
}
|
||||
}
|
||||
|
||||
function getMetricUnit(metric) {
|
||||
return metric === 'speed' ? 'ms' : 'MB';
|
||||
}
|
||||
@@ -0,0 +1,96 @@
|
||||
body {
|
||||
font-family: Arial, sans-serif;
|
||||
background-color: #f4f4f9;
|
||||
margin: 0;
|
||||
padding: 0;
|
||||
}
|
||||
|
||||
.container {
|
||||
max-width: 80vw;
|
||||
margin: 2rem auto;
|
||||
padding: 1rem;
|
||||
padding-left: 10px;
|
||||
padding-right: 10px;
|
||||
background: white;
|
||||
border-radius: 10px;
|
||||
box-shadow: 0 0 10px rgba(0, 0, 0, 0.1);
|
||||
display: flex;
|
||||
justify-content: space-between;
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
h1 {
|
||||
text-align: center;
|
||||
margin-bottom: 2rem;
|
||||
}
|
||||
|
||||
h2 {
|
||||
text-align: center;
|
||||
margin: 1rem 0;
|
||||
color: #333;
|
||||
}
|
||||
|
||||
.controls {
|
||||
display: flex;
|
||||
flex-wrap: wrap;
|
||||
justify-content: center;
|
||||
gap: 1rem;
|
||||
margin-bottom: 2rem;
|
||||
background: #f8f9fa;
|
||||
padding: 15px;
|
||||
border-radius: 8px;
|
||||
border: 1px solid #e9ecef;
|
||||
}
|
||||
|
||||
.controls label {
|
||||
display: block;
|
||||
margin-bottom: 5px;
|
||||
color: #495057;
|
||||
font-size: 14px;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
.metric-section {
|
||||
margin-top: 2rem;
|
||||
}
|
||||
|
||||
.metric-section canvas {
|
||||
max-width: 100%;
|
||||
height: 300px;
|
||||
margin-bottom: 2rem;
|
||||
}
|
||||
|
||||
canvas {
|
||||
max-width: 100%;
|
||||
height: 500px;
|
||||
}
|
||||
|
||||
.panel1 {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
.panel2 {
|
||||
width: 100%;
|
||||
}
|
||||
|
||||
/* Select styling improvements */
|
||||
select {
|
||||
padding: 8px 12px;
|
||||
border: 2px solid #e9ecef;
|
||||
border-radius: 6px;
|
||||
background: white;
|
||||
font-size: 14px;
|
||||
color: #333;
|
||||
transition: border-color 0.3s ease;
|
||||
min-width: 150px;
|
||||
}
|
||||
|
||||
select:focus {
|
||||
outline: none;
|
||||
border-color: #667eea;
|
||||
box-shadow: 0 0 0 3px rgba(102, 126, 234, 0.1);
|
||||
}
|
||||
|
||||
select:hover {
|
||||
border-color: #667eea;
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"description": "Golden reference commits for kernel performance benchmarking",
|
||||
"default_commit": "c1bdc24",
|
||||
"golden_commits": {
|
||||
"cross_entropy": "0701bab",
|
||||
"distill_jsd_loss": "c1bdc24",
|
||||
"dpo_loss": "c1bdc24",
|
||||
"dyt_beta=False": "0701bab",
|
||||
"dyt_beta=True": "0701bab",
|
||||
"fused_linear_cpo_loss": "c1bdc24",
|
||||
"fused_linear_cross_entropy": "0701bab",
|
||||
"fused_linear_jsd": "c1bdc24",
|
||||
"fused_linear_simpo_loss": "c1bdc24",
|
||||
"fused_neighborhood_attention": "0701bab",
|
||||
"geglu": "0701bab",
|
||||
"group_norm": "0701bab",
|
||||
"jsd": "c1bdc24",
|
||||
"kl_div": "0701bab",
|
||||
"kto_loss": "c1bdc24",
|
||||
"layer_norm": "0701bab",
|
||||
"multi_token_attention": "0701bab",
|
||||
"rms_norm": "0701bab",
|
||||
"rope": "0701bab",
|
||||
"softmax": "0701bab",
|
||||
"sparse_multi_token_attention": "0701bab",
|
||||
"sparsemax": "0701bab",
|
||||
"swiglu": "0701bab",
|
||||
"tvd": "0701bab",
|
||||
"distill_cosine_loss": "08906d3"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
|
||||
<meta charset="UTF-8">
|
||||
<title>Liger Benchmark Dashboard</title>
|
||||
<link rel="stylesheet" href="style.css">
|
||||
</head>
|
||||
<body data-theme="light">
|
||||
<div class="container">
|
||||
<button class="theme-toggle" onclick="toggleTheme()" title="Toggle Theme">☀️</button>
|
||||
<button class="theme-toggle" onclick="goToDetailedView()">Detailed View</button>
|
||||
<h1>Liger Kernel Benchmark Comparison</h1>
|
||||
|
||||
<div class="selectors">
|
||||
<label>
|
||||
Show last N commits:
|
||||
<select id="commitCount" onchange="loadData()">
|
||||
<option value="5">5</option>
|
||||
<option value="20">20</option>
|
||||
<option value="50">50</option>
|
||||
<option value="all">All</option>
|
||||
</select>
|
||||
</label>
|
||||
<label>
|
||||
Metric type:
|
||||
<select id="metricType" onchange="renderTables()">
|
||||
<option value="full">full</option>
|
||||
<option value="forward">forward</option>
|
||||
<option value="backward">backward</option>
|
||||
</select>
|
||||
</label>
|
||||
<label>
|
||||
Category:
|
||||
<select id="categoryFilter" onchange="renderTables()">
|
||||
<option value="all">All Categories</option>
|
||||
<option value="pretraining">Pretraining</option>
|
||||
<option value="distillation">Distillation</option>
|
||||
<option value="post_training">Post Training</option>
|
||||
</select>
|
||||
</label>
|
||||
|
||||
<label>
|
||||
Show:
|
||||
<select id="viewType" onchange="loadData()">
|
||||
<option value="commits">Commits</option>
|
||||
<option value="versions">Versions</option>
|
||||
</select>
|
||||
</label>
|
||||
|
||||
<input type="text" id="kernelSearch" placeholder="Search kernel name..." oninput="renderTables()">
|
||||
</div>
|
||||
|
||||
<div class="legend">
|
||||
<h3>Kernel Categories</h3>
|
||||
<div class="legend-items">
|
||||
<div class="legend-item">
|
||||
<span class="legend-color pretraining"></span>
|
||||
<span class="legend-label">Pretraining</span>
|
||||
</div>
|
||||
<div class="legend-item">
|
||||
<span class="legend-color distillation"></span>
|
||||
<span class="legend-label">Distillation</span>
|
||||
</div>
|
||||
<div class="legend-item">
|
||||
<span class="legend-color post_training"></span>
|
||||
<span class="legend-label">Post Training</span>
|
||||
</div>
|
||||
<div class="legend-item">
|
||||
<span class="legend-color other"></span>
|
||||
<span class="legend-label">Other</span>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<h2>Memory</h2>
|
||||
<div class="scroll-container">
|
||||
<table id="memoryTable"></table>
|
||||
</div>
|
||||
|
||||
<h2>Speed</h2>
|
||||
<div class="scroll-container">
|
||||
<table id="speedTable"></table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="script.js"></script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -0,0 +1,425 @@
|
||||
// TODO: Make this configurable into a separate file
|
||||
const defaultReferenceCommit = 'c1bdc24';
|
||||
const benchmarkPrimaryBase = 'https://raw.githubusercontent.com/linkedin/Liger-Kernel/refs/heads/gh-pages/benchmarks';
|
||||
const benchmarkFallbackBase = 'https://cdn.jsdelivr.net/gh/linkedin/Liger-Kernel@gh-pages/benchmarks';
|
||||
|
||||
// Kernel categories configuration
|
||||
const kernelCategories = {
|
||||
"pretraining": [
|
||||
"cross_entropy",
|
||||
"fused_linear_cross_entropy",
|
||||
"softmax",
|
||||
"sparsemax",
|
||||
"rope",
|
||||
"geglu",
|
||||
"swiglu",
|
||||
"layer_norm",
|
||||
"rms_norm",
|
||||
"group_norm",
|
||||
"multi_token_attention",
|
||||
"sparse_multi_token_attention",
|
||||
"fused_neighborhood_attention"
|
||||
],
|
||||
"distillation": [
|
||||
"distill_jsd_loss",
|
||||
"distill_cosine_loss",
|
||||
"jsd",
|
||||
"kl_div",
|
||||
"tvd",
|
||||
"fused_linear_jsd",
|
||||
"dyt_beta=True",
|
||||
"dyt_beta=False"
|
||||
],
|
||||
"post_training": [
|
||||
"dpo_loss",
|
||||
"kto_loss",
|
||||
"fused_linear_cpo_loss",
|
||||
"fused_linear_simpo_loss"
|
||||
]
|
||||
};
|
||||
|
||||
let allDataByCommit = {};
|
||||
let kernelMeta = {};
|
||||
let configData = {}; // Store configuration data for tooltips - format: {commit: {kernelKey: config}}
|
||||
let goldenCommits = {};
|
||||
|
||||
async function fetchWithFallback(path, responseType = 'text') {
|
||||
const urls = [
|
||||
`${benchmarkPrimaryBase}/${path}`,
|
||||
`${benchmarkFallbackBase}/${path}`,
|
||||
];
|
||||
let lastErr;
|
||||
for (const url of urls) {
|
||||
try {
|
||||
const res = await fetch(url);
|
||||
if (res.ok) {
|
||||
if (responseType === 'json') return await res.json();
|
||||
return await res.text();
|
||||
}
|
||||
lastErr = new Error(`HTTP ${res.status} for ${url}`);
|
||||
} catch (e) {
|
||||
lastErr = e;
|
||||
}
|
||||
}
|
||||
throw lastErr || new Error('Failed to fetch with fallback');
|
||||
}
|
||||
|
||||
async function loadGoldenCommits() {
|
||||
try {
|
||||
const data = await fetchWithFallback('golden_commits.json', 'json');
|
||||
return data.golden_commits || {};
|
||||
} catch (e) {
|
||||
console.warn('Could not load golden commits from JSON');
|
||||
}
|
||||
return {};
|
||||
}
|
||||
|
||||
function getGoldenCommit(kernelName) {
|
||||
// Extract base kernel name by removing operation mode and metric suffixes
|
||||
let baseKernelName = kernelName;
|
||||
|
||||
const suffixes = ['_forward_speed', '_backward_speed', '_full_speed', '_full_memory', '_forward_memory', '_backward_memory'];
|
||||
for (const suffix of suffixes) {
|
||||
if (baseKernelName.endsWith(suffix)) {
|
||||
baseKernelName = baseKernelName.slice(0, -suffix.length);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Check for exact base kernel match
|
||||
if (goldenCommits[baseKernelName]) {
|
||||
return goldenCommits[baseKernelName];
|
||||
}
|
||||
|
||||
// Check for partial matches - find the longest matching key
|
||||
let bestMatch = '';
|
||||
let bestCommit = defaultReferenceCommit;
|
||||
|
||||
for (const [key, commit] of Object.entries(goldenCommits)) {
|
||||
if (kernelName.includes(key) && key.length > bestMatch.length) {
|
||||
bestMatch = key;
|
||||
bestCommit = commit;
|
||||
}
|
||||
}
|
||||
|
||||
return bestCommit;
|
||||
}
|
||||
|
||||
function getKernelCategory(kernelName) {
|
||||
// Extract base kernel name by removing operation mode and metric suffixes
|
||||
let baseKernelName = kernelName;
|
||||
|
||||
const suffixes = ['_forward_speed', '_backward_speed', '_full_speed', '_full_memory', '_forward_memory', '_backward_memory'];
|
||||
for (const suffix of suffixes) {
|
||||
if (baseKernelName.endsWith(suffix)) {
|
||||
baseKernelName = baseKernelName.slice(0, -suffix.length);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// Find which category this kernel belongs to
|
||||
for (const [category, kernels] of Object.entries(kernelCategories)) {
|
||||
for (const kernelPattern of kernels) {
|
||||
if (baseKernelName === kernelPattern || baseKernelName.includes(kernelPattern)) {
|
||||
return category;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return 'other'; // Default category for unmatched kernels
|
||||
}
|
||||
|
||||
function toggleTheme() {
|
||||
const body = document.body;
|
||||
const themeButton = document.querySelector('.theme-toggle');
|
||||
const isDark = body.getAttribute('data-theme') === 'dark';
|
||||
body.setAttribute('data-theme', isDark ? 'light' : 'dark');
|
||||
themeButton.textContent = isDark ? '☀️' : '🌙';
|
||||
}
|
||||
|
||||
function goToDetailedView() {
|
||||
const baseUrl = window.location.href.endsWith('/') ? window.location.href : window.location.href + '/';
|
||||
window.location.href = baseUrl + 'detailed';
|
||||
}
|
||||
|
||||
async function fetchCommits() {
|
||||
const text = await fetchWithFallback('commits.txt', 'text');
|
||||
const all = text.trim().split('\n');
|
||||
const commits = all.filter(line => !line.includes('-'));
|
||||
const versions = all.filter(line => line.includes('-'));
|
||||
return { commits, versions };
|
||||
}
|
||||
|
||||
async function fetchCSV(commit) {
|
||||
const text = await fetchWithFallback(`${commit}/benchmark.csv`, 'text');
|
||||
const rows = text.trim().split('\n');
|
||||
const headers = rows[0].split(',');
|
||||
const data = {};
|
||||
|
||||
for (let i = 1; i < rows.length; i++) {
|
||||
const rowText = rows[i];
|
||||
const cols = [];
|
||||
let currentCol = '';
|
||||
let inQuotes = false;
|
||||
let quoteChar = '';
|
||||
|
||||
// Parse CSV row properly handling quoted fields
|
||||
for (let j = 0; j < rowText.length; j++) {
|
||||
const char = rowText[j];
|
||||
|
||||
if ((char === '"' || char === "'") && !inQuotes) {
|
||||
inQuotes = true;
|
||||
quoteChar = char;
|
||||
} else if (char === quoteChar && inQuotes) {
|
||||
inQuotes = false;
|
||||
quoteChar = '';
|
||||
} else if (char === ',' && !inQuotes) {
|
||||
cols.push(currentCol.trim());
|
||||
currentCol = '';
|
||||
} else {
|
||||
currentCol += char;
|
||||
}
|
||||
}
|
||||
cols.push(currentCol.trim());
|
||||
|
||||
const row = Object.fromEntries(headers.map((h, idx) => [h.trim(), cols[idx]?.trim()]));
|
||||
if (row.kernel_provider !== 'liger') continue;
|
||||
|
||||
const kernelKey = `${row.kernel_name}_${row.kernel_operation_mode}_${row.metric_name}`;
|
||||
|
||||
if (!data[kernelKey]) {
|
||||
data[kernelKey] = parseFloat(row.y_value_50);
|
||||
if (!kernelMeta[kernelKey]) {
|
||||
kernelMeta[kernelKey] = {
|
||||
x_label: row.x_label,
|
||||
x_value: row.x_value
|
||||
};
|
||||
}
|
||||
// Store configuration data for tooltips
|
||||
if (!configData[commit]) {
|
||||
configData[commit] = {};
|
||||
}
|
||||
if (!configData[commit][kernelKey]) {
|
||||
configData[commit][kernelKey] = {
|
||||
extra_benchmark_config_str: row.extra_benchmark_config_str || 'N/A',
|
||||
gpu_name: row.gpu_name || 'N/A',
|
||||
liger_version: row.liger_version || 'N/A'
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
async function loadData() {
|
||||
document.getElementById('speedTable').innerHTML = 'Loading...';
|
||||
document.getElementById('memoryTable').innerHTML = 'Loading...';
|
||||
|
||||
goldenCommits = await loadGoldenCommits();
|
||||
|
||||
const count = document.getElementById('commitCount').value;
|
||||
const viewType = document.getElementById('viewType').value;
|
||||
const { commits, versions } = await fetchCommits();
|
||||
|
||||
let selectedList = viewType === 'versions' ? versions : commits;
|
||||
const selectedCommits = count === 'all' ? selectedList : selectedList.slice(-parseInt(count));
|
||||
|
||||
// Collect all unique commits that need to be loaded
|
||||
const commitsToLoad = new Set(selectedCommits);
|
||||
|
||||
// Add reference commits only if they're not already in the selected commits
|
||||
Object.values(goldenCommits).forEach(commit => {
|
||||
if (!selectedCommits.includes(commit)) {
|
||||
commitsToLoad.add(commit);
|
||||
}
|
||||
});
|
||||
|
||||
// Also add the default reference commit if not already included
|
||||
if (!selectedCommits.includes(defaultReferenceCommit)) {
|
||||
commitsToLoad.add(defaultReferenceCommit);
|
||||
}
|
||||
|
||||
allDataByCommit = {};
|
||||
kernelMeta = {};
|
||||
configData = {};
|
||||
|
||||
await Promise.all([...commitsToLoad].map(async (commit) => {
|
||||
const data = await fetchCSV(commit);
|
||||
if (data) allDataByCommit[commit] = data;
|
||||
}));
|
||||
|
||||
await renderTables();
|
||||
}
|
||||
|
||||
function compareMetric(current, reference) {
|
||||
const THRESHOLD = 0.10;
|
||||
if (current == null || reference == null) return '';
|
||||
const delta = (current - reference) / reference;
|
||||
if (delta > THRESHOLD) return 'red';
|
||||
if (delta < 0) return 'green';
|
||||
return '';
|
||||
}
|
||||
|
||||
function formatConfigForTooltip(config) {
|
||||
try {
|
||||
const parsed = JSON.parse(config.extra_benchmark_config_str);
|
||||
const configStr = Object.entries(parsed)
|
||||
.map(([key, value]) => `${key}: ${value}`)
|
||||
.join('\n');
|
||||
return `Extra Config:\n${configStr}\n\nGPU: ${config.gpu_name}\nLiger Version: ${config.liger_version}`;
|
||||
} catch (e) {
|
||||
return `Extra Config: ${config.extra_benchmark_config_str}\n\nGPU: ${config.gpu_name}\nLiger Version: ${config.liger_version}`;
|
||||
}
|
||||
}
|
||||
|
||||
async function renderTables() {
|
||||
const metricType = document.getElementById('metricType').value;
|
||||
const searchQuery = document.getElementById('kernelSearch').value.toLowerCase();
|
||||
const categoryFilter = document.getElementById('categoryFilter').value;
|
||||
const speedTable = document.getElementById('speedTable');
|
||||
const memoryTable = document.getElementById('memoryTable');
|
||||
const viewType = document.getElementById('viewType').value;
|
||||
|
||||
speedTable.innerHTML = '';
|
||||
memoryTable.innerHTML = '';
|
||||
|
||||
const count = document.getElementById('commitCount').value;
|
||||
const { commits: allCommitsList, versions: allVersionsList } = await fetchCommits();
|
||||
const selectedList = viewType === 'versions' ? allVersionsList : allCommitsList;
|
||||
const selectedCommits = count === 'all' ? selectedList : selectedList.slice(-parseInt(count));
|
||||
|
||||
const allLoadedCommits = Object.keys(allDataByCommit);
|
||||
const commits = selectedCommits.filter(commit => allLoadedCommits.includes(commit)).reverse();
|
||||
|
||||
function createTable(table, filterMetric) {
|
||||
let header = `<tr><th>Kernel</th><th>X</th>`;
|
||||
if (viewType === 'commits') {
|
||||
header += `<th>Golden Reference</th>`;
|
||||
}
|
||||
for (const commit of commits) {
|
||||
header += `<th>${commit}</th>`;
|
||||
}
|
||||
header += '</tr>';
|
||||
table.innerHTML += header;
|
||||
|
||||
const allKeys = new Set();
|
||||
for (const commit of commits) {
|
||||
for (const k in allDataByCommit[commit]) {
|
||||
if (k.includes(`_${metricType}_${filterMetric}`)) allKeys.add(k);
|
||||
}
|
||||
}
|
||||
|
||||
[...allKeys].forEach((kernelKey) => {
|
||||
if (searchQuery && !kernelKey.toLowerCase().includes(searchQuery)) return;
|
||||
|
||||
if (categoryFilter !== 'all') {
|
||||
const kernelCategory = getKernelCategory(kernelKey);
|
||||
if (kernelCategory !== categoryFilter) return;
|
||||
}
|
||||
|
||||
const kernelCategory = getKernelCategory(kernelKey);
|
||||
let row = `<tr class="category-${kernelCategory}">`;
|
||||
row += `<td>${kernelKey}</td><td>${(kernelMeta[kernelKey]?.x_label || '?')}=${(kernelMeta[kernelKey]?.x_value || '?')}</td>`;
|
||||
if (viewType === 'commits') {
|
||||
const goldenCommit = getGoldenCommit(kernelKey);
|
||||
let refVal = allDataByCommit[goldenCommit]?.[kernelKey];
|
||||
if (refVal == null) {
|
||||
const versionedKey = Object.keys(allDataByCommit).find(k => k.startsWith(goldenCommit + '-'));
|
||||
if (versionedKey && allDataByCommit[versionedKey]?.[kernelKey] != null) {
|
||||
refVal = allDataByCommit[versionedKey][kernelKey];
|
||||
}
|
||||
}
|
||||
const refConfig = configData[goldenCommit]?.[kernelKey] || { extra_benchmark_config_str: 'N/A', gpu_name: 'N/A', liger_version: 'N/A' };
|
||||
const refTooltipText = formatConfigForTooltip(refConfig) + `\n\nCategory: ${kernelCategory}\nGolden Commit: ${goldenCommit}`;
|
||||
row += `<td title="${goldenCommit}" data-tooltip="${refTooltipText}" style="cursor: pointer;">${refVal != null ? refVal.toFixed(2) : 'N/A'}</td>`;
|
||||
}
|
||||
for (const commit of commits) {
|
||||
const val = allDataByCommit[commit]?.[kernelKey];
|
||||
let cellClass = '';
|
||||
if (viewType === 'commits') {
|
||||
const goldenCommit = getGoldenCommit(kernelKey);
|
||||
const refVal = allDataByCommit[goldenCommit]?.[kernelKey];
|
||||
const cls = compareMetric(val, refVal);
|
||||
const isGolden = commit === goldenCommit;
|
||||
cellClass = isGolden ? 'golden' : cls;
|
||||
}
|
||||
|
||||
const config = configData[commit]?.[kernelKey] || { extra_benchmark_config_str: 'N/A', gpu_name: 'N/A', liger_version: 'N/A' };
|
||||
const tooltipText = formatConfigForTooltip(config) + `\n\nCategory: ${kernelCategory}\nCommit: ${commit}`;
|
||||
row += `<td class="${cellClass}" data-tooltip="${tooltipText}" style="cursor: pointer;">${val != null ? val.toFixed(2) : 'N/A'}</td>`;
|
||||
}
|
||||
row += '</tr>';
|
||||
table.innerHTML += row;
|
||||
});
|
||||
}
|
||||
|
||||
createTable(memoryTable, 'memory');
|
||||
createTable(speedTable, 'speed');
|
||||
|
||||
addTooltipListeners();
|
||||
}
|
||||
|
||||
let tooltipEl;
|
||||
|
||||
function addTooltipListeners() {
|
||||
if (!tooltipEl) {
|
||||
tooltipEl = document.createElement('div');
|
||||
tooltipEl.id = 'custom-tooltip';
|
||||
tooltipEl.className = 'custom-tooltip';
|
||||
document.body.appendChild(tooltipEl);
|
||||
}
|
||||
|
||||
const tooltipCells = document.querySelectorAll('td[data-tooltip]');
|
||||
|
||||
tooltipCells.forEach(cell => {
|
||||
cell.addEventListener('mouseenter', showTooltip);
|
||||
cell.addEventListener('mousemove', moveTooltip);
|
||||
cell.addEventListener('mouseleave', hideTooltip);
|
||||
});
|
||||
}
|
||||
|
||||
function showTooltip(event) {
|
||||
tooltipEl.textContent = event.currentTarget.getAttribute('data-tooltip');
|
||||
tooltipEl.style.display = 'block';
|
||||
moveTooltip(event);
|
||||
}
|
||||
|
||||
function moveTooltip(event) {
|
||||
if (!tooltipEl || tooltipEl.style.display === 'none') return;
|
||||
|
||||
const viewportWidth = window.innerWidth;
|
||||
const viewportHeight = window.innerHeight;
|
||||
|
||||
let left = event.clientX + 15;
|
||||
let top = event.clientY + 15;
|
||||
|
||||
const tooltipWidth = tooltipEl.offsetWidth;
|
||||
const tooltipHeight = tooltipEl.offsetHeight;
|
||||
|
||||
if (left + tooltipWidth > viewportWidth) {
|
||||
left = event.clientX - tooltipWidth - 15;
|
||||
}
|
||||
|
||||
if (top + tooltipHeight > viewportHeight) {
|
||||
top = event.clientY - tooltipHeight - 15;
|
||||
}
|
||||
|
||||
if (left < 0) {
|
||||
left = 5;
|
||||
}
|
||||
|
||||
if (top < 0) {
|
||||
top = 5;
|
||||
}
|
||||
|
||||
tooltipEl.style.left = `${left}px`;
|
||||
tooltipEl.style.top = `${top}px`;
|
||||
}
|
||||
|
||||
function hideTooltip() {
|
||||
if (tooltipEl) {
|
||||
tooltipEl.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
window.onload = loadData;
|
||||
@@ -0,0 +1,252 @@
|
||||
:root {
|
||||
--primary-color: #4CAF50;
|
||||
--danger-color: #f44336;
|
||||
--success-color: #4caf50;
|
||||
--font-color: #333;
|
||||
--bg-color: #ffffff;
|
||||
--secondary-color: #f2f2f2;
|
||||
}
|
||||
|
||||
[data-theme="dark"] {
|
||||
--primary-color: #4CAF50;
|
||||
--danger-color: #e57373;
|
||||
--success-color: #81c784;
|
||||
--font-color: #f1f1f1;
|
||||
--bg-color: #1e1e1e;
|
||||
--secondary-color: #2c2c2c;
|
||||
}
|
||||
|
||||
[data-theme="dark"] td.golden {
|
||||
background-color: #3a3a1a;
|
||||
color: #ffc107;
|
||||
border-color: #b8860b;
|
||||
}
|
||||
|
||||
body {
|
||||
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif;
|
||||
margin: 0;
|
||||
padding: 20px;
|
||||
background-color: var(--bg-color);
|
||||
color: var(--font-color);
|
||||
text-align: center;
|
||||
transition: background-color 0.3s, color 0.3s;
|
||||
}
|
||||
|
||||
.container {
|
||||
display: inline-block;
|
||||
text-align: left;
|
||||
max-width: 95%;
|
||||
background-color: var(--bg-color);
|
||||
padding: 20px;
|
||||
border-radius: 12px;
|
||||
box-shadow: 0 0 15px rgba(0, 0, 0, 0.1);
|
||||
transition: background-color 0.3s;
|
||||
}
|
||||
|
||||
h1, h2 {
|
||||
color: var(--primary-color);
|
||||
}
|
||||
|
||||
table {
|
||||
border-collapse: collapse;
|
||||
width: 100%;
|
||||
margin-top: 10px;
|
||||
background-color: var(--bg-color);
|
||||
border-radius: 8px;
|
||||
overflow: hidden;
|
||||
}
|
||||
|
||||
th, td {
|
||||
border: 1px solid #ccc;
|
||||
padding: 12px 8px;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
th {
|
||||
background-color: var(--secondary-color);
|
||||
position: sticky;
|
||||
top: 0;
|
||||
z-index: 2;
|
||||
}
|
||||
|
||||
td.green {
|
||||
background-color: #d4edda;
|
||||
color: #155724;
|
||||
}
|
||||
|
||||
td.red {
|
||||
background-color: #f8d7da;
|
||||
color: #721c24;
|
||||
}
|
||||
|
||||
td.golden {
|
||||
background-color: #fff3cd;
|
||||
color: #856404;
|
||||
font-weight: bold;
|
||||
border: 2px solid #ffc107;
|
||||
}
|
||||
|
||||
/* Tooltip styles */
|
||||
tr[data-tooltip]:hover {
|
||||
background-color: rgba(76, 175, 80, 0.1) !important;
|
||||
}
|
||||
|
||||
.custom-tooltip {
|
||||
position: fixed;
|
||||
display: none;
|
||||
background-color: #333;
|
||||
color: white;
|
||||
padding: 8px 12px;
|
||||
border-radius: 6px;
|
||||
font-size: 12px;
|
||||
white-space: pre-line;
|
||||
z-index: 10001;
|
||||
max-width: 300px;
|
||||
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.3);
|
||||
pointer-events: none;
|
||||
}
|
||||
|
||||
.selectors {
|
||||
margin-bottom: 20px;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.selectors label {
|
||||
margin: 0 10px;
|
||||
}
|
||||
|
||||
#kernelSearch {
|
||||
padding: 8px;
|
||||
margin: 10px 0;
|
||||
width: 300px;
|
||||
font-size: 16px;
|
||||
border-radius: 5px;
|
||||
border: 1px solid #ccc;
|
||||
}
|
||||
|
||||
button {
|
||||
padding: 8px 16px;
|
||||
background-color: var(--primary-color);
|
||||
color: #fff;
|
||||
border: none;
|
||||
border-radius: 5px;
|
||||
cursor: pointer;
|
||||
font-weight: bold;
|
||||
}
|
||||
|
||||
button:hover {
|
||||
background-color: #388e3c;
|
||||
}
|
||||
|
||||
.scroll-container {
|
||||
overflow-x: auto;
|
||||
margin-bottom: 40px;
|
||||
}
|
||||
|
||||
.theme-toggle {
|
||||
float: right;
|
||||
margin-bottom: 20px;
|
||||
cursor: pointer;
|
||||
font-weight: bold;
|
||||
background: none;
|
||||
border: none;
|
||||
color: var(--font-color);
|
||||
font-size: 20px;
|
||||
padding: 8px;
|
||||
border-radius: 50%;
|
||||
transition: background-color 0.3s;
|
||||
}
|
||||
|
||||
.theme-toggle:hover {
|
||||
background-color: var(--secondary-color);
|
||||
}
|
||||
|
||||
/* Category-based row styling */
|
||||
.category-pretraining {
|
||||
border-left: 4px solid #2196F3;
|
||||
}
|
||||
|
||||
.category-distillation {
|
||||
border-left: 4px solid #FF9800;
|
||||
}
|
||||
|
||||
.category-post_training {
|
||||
border-left: 4px solid #9C27B0;
|
||||
}
|
||||
|
||||
.category-other {
|
||||
border-left: 4px solid #607D8B;
|
||||
}
|
||||
|
||||
/* Category filter styling */
|
||||
#categoryFilter {
|
||||
padding: 8px;
|
||||
border-radius: 5px;
|
||||
border: 1px solid #ccc;
|
||||
background-color: var(--bg-color);
|
||||
color: var(--font-color);
|
||||
}
|
||||
|
||||
#viewType {
|
||||
padding: 8px;
|
||||
border-radius: 5px;
|
||||
border: 1px solid #ccc;
|
||||
background-color: var(--bg-color);
|
||||
color: var(--font-color);
|
||||
margin-left: 10px;
|
||||
}
|
||||
|
||||
|
||||
/* Legend styling */
|
||||
.legend {
|
||||
margin: 20px 0;
|
||||
text-align: center;
|
||||
}
|
||||
|
||||
.legend h3 {
|
||||
margin-bottom: 10px;
|
||||
color: var(--font-color);
|
||||
font-size: 16px;
|
||||
}
|
||||
|
||||
.legend-items {
|
||||
display: flex;
|
||||
justify-content: center;
|
||||
flex-wrap: wrap;
|
||||
gap: 20px;
|
||||
}
|
||||
|
||||
.legend-item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 8px;
|
||||
}
|
||||
|
||||
.legend-color {
|
||||
width: 20px;
|
||||
height: 20px;
|
||||
border-radius: 3px;
|
||||
border: 1px solid #ddd;
|
||||
}
|
||||
|
||||
.legend-color.pretraining {
|
||||
background-color: #2196F3;
|
||||
}
|
||||
|
||||
.legend-color.distillation {
|
||||
background-color: #FF9800;
|
||||
}
|
||||
|
||||
.legend-color.post_training {
|
||||
background-color: #9C27B0;
|
||||
}
|
||||
|
||||
.legend-color.other {
|
||||
background-color: #607D8B;
|
||||
}
|
||||
|
||||
.legend-label {
|
||||
font-size: 14px;
|
||||
font-weight: 500;
|
||||
color: var(--font-color);
|
||||
}
|
||||
Reference in New Issue
Block a user