Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Docs] Point to smoothquant guide after failure to match #1074

Closed
wants to merge 1 commit into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 21 additions & 10 deletions src/llmcompressor/modifiers/smoothquant/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -176,25 +176,36 @@ def _resolve_mappings(self, model: Module) -> List:
would match model.layer.0.p_proj to model.layer.0.self_attn_layer_norm and
repeat for model.layer.1 and so on
"""
resolved_mappings = []
for to_balance, to_smooth in self.mappings:
to_smooth_layers = get_layers(to_smooth, model)
for layer_name, smooth_layer in to_smooth_layers.items():
if not match_targets(layer_name, self.ignore)[0]:
try:
resolved_mappings = []
for to_balance, to_smooth in self.mappings:
to_smooth_layers = get_layers(to_smooth, model)
for layer_name, smooth_layer in to_smooth_layers.items():
if match_targets(layer_name, self.ignore)[0]:
continue
balance_layers = []
for balance_suffix in to_balance:
# find the submodule that matches the activation layer
_, balance_layer = get_matching_layer(
balance_suffix, layer_name, model
)
if balance_layer:
balance_layers.append(balance_layer)
res = get_matching_layer(balance_suffix, layer_name, model)
if res is None:
raise ValueError(
f"fCould not find target {balance_suffix} "
f"in module {type(model)}"
)
balance_layer = res[1]
balance_layers.append(balance_layer)
# each mapping can contain multiple layers to balance, but only
# one layer to smooth
mapping = SmoothQuantMapping(
layer_name, smooth_layer, balance_layers
)
resolved_mappings.append(mapping)
except ValueError as exception:
raise ValueError(
f"Failed to resolve smoothquant mappings `{exception}`. "
"For more information regarding adding your own smoothquant mappings, "
"see `src/llmcompressor/modifiers/smoothquant/README.md`"
)
return resolved_mappings

def _setup_scale_hooks(self):
Expand Down
2 changes: 1 addition & 1 deletion src/llmcompressor/utils/pytorch/module.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,7 +157,7 @@ def match_layers_params(

missed = [target for found, target in zip(targets_found, targets) if not found]
if len(missed) > 0:
raise ValueError(f"Could not find targets {missed} in module {module}")
raise ValueError(f"Could not find targets {missed} in module {type(module)}")

return resolved

Expand Down
Loading