Skip to content

Commit

Permalink
subtract num suffix tokens from mask len
Browse files Browse the repository at this point in the history
  • Loading branch information
bastiscode committed Jun 28, 2024
1 parent fe95ebb commit 2d793e7
Showing 1 changed file with 1 addition and 1 deletion.
2 changes: 1 addition & 1 deletion src/data/task.rs
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ fn generation_input(
Box::new(move |item| {
let mask_len = if mask_prefix {
let pfx = format!("{}{}", item.input, separator.as_deref().unwrap_or_default());
tokenizer.tokenize(&pfx, false)?.token_ids.len()
tokenizer.tokenize(&pfx, false)?.token_ids.len() - tokenizer.num_suffix_tokens()
} else {
0
};
Expand Down

0 comments on commit 2d793e7

Please sign in to comment.