-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathllama_3_2_1b_hf_recipe.py
19 lines (18 loc) · 1.24 KB
/
llama_3_2_1b_hf_recipe.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
llama_recipe = {
# 'model.layers.0.mlp':{'pr':0.2}, 'model.layers.0.self_attn':{'pr':0.2},
# 'model.layers.1.mlp':{'pr':0.2}, 'model.layers.1.self_attn':{'pr':0.2},
# 'model.layers.2.mlp':{'pr':0.2}, 'model.layers.2.self_attn':{'pr':0.2},
# 'model.layers.3.mlp':{'pr':0.2}, 'model.layers.3.self_attn':{'pr':0.2},
# 'model.layers.4.mlp':{'pr':0.2}, 'model.layers.4.self_attn':{'pr':0.2},
# 'model.layers.5.mlp':{'pr':0.2}, 'model.layers.5.self_attn':{'pr':0.2},
# 'model.layers.6.mlp':{'pr':0.8}, 'model.layers.6.self_attn':{'pr':0.2},
# 'model.layers.7.mlp':{'pr':0.8}, 'model.layers.7.self_attn':{'pr':0.2},
# 'model.layers.8.mlp':{'pr':0.8}, 'model.layers.8.self_attn':{'pr':0.2},
# 'model.layers.9.mlp':{'pr':0.8}, 'model.layers.9.self_attn':{'pr':0.2},
# 'model.layers.10.mlp':{'pr':0.8}, 'model.layers.10.self_attn':{'pr':0.2},
# 'model.layers.11.mlp':{'pr':0.8}, 'model.layers.11.self_attn':{'pr':0.2},
'model.layers.12.mlp':{'pr':0.8}, 'model.layers.12.self_attn':{'pr':0.2},
'model.layers.13.mlp':{'pr':0.8}, 'model.layers.13.self_attn':{'pr':0.2},
'model.layers.14.mlp':{'pr':0.8}, 'model.layers.14.self_attn':{'pr':0.2},
# 'model.layers.15.mlp':{'pr':0.8}, 'model.layers.15.self_attn':{'pr':0.2},
}